Merge pull request #58 from BBVA/develop

Add multi branches feature

.env.required

@@ -1,6 +1,6 @@
-BROKER_URI
-DATABASE_URI
-SHARED_VOLUME_PATH
-PLUGINS_LOCATION
-LOCAL_PRIVATE_KEY_FILE
-LOG_LEVEL
+BROKER_URI (example: redis://localhost:6379)
+DATABASE_URI (example: postgresql://postgres:postgres@localhost:5433/deeptracy)
+SHARED_VOLUME_PATH (example: /tmp/deeptracy)
+PLUGINS_LOCATION (example: /opt/deeptracy/plugins)
+LOCAL_PRIVATE_KEY_FILE (example: /root/.ssh/id_rsa)
+LOG_LEVEL (example: DEBUG)

deeptracy/tasks/merge_results.py

@@ -50,7 +50,11 @@ def merge_results(results, scan_id=None):
         scan = get_scan(scan_id, session)
         project = scan.project
 
+        # TODO: implement merge_results
+        vulnerabilities = []
+
         scan = update_scan_state(scan, ScanState.DONE, session)
+        scan.total_vulnerabilities = len(vulnerabilities)
         session.commit()
 
         if project.hook_type != ProjectHookType.NONE.name:

deeptracy/tasks/prepare_scan.py

@@ -28,7 +28,6 @@
 from .base_task import TaskException, DeeptracyTask
 from .scan_deps import scan_deps
 
-
 logger = get_task_logger('deeptracy')
 
 
@@ -41,25 +40,35 @@ def prepare_scan(scan_id: str):
 
         logger.debug('{} for project({})'.format(scan_id, project.id))
 
-        # clone the repository in a shared volume
-        cloned_dir = clone_project(config.SHARED_VOLUME_PATH, scan_id, project.repo, project.repo_auth_type)
-        scan.source_path = cloned_dir
-        session.add(scan)
-        logger.debug('{} cloned dir {}'.format(scan_id, cloned_dir))
-
-        # if a .deeprtacy.yml is found, parse it to a dictionary
+        try:
+            # clone the repository in a shared volume
+            cloned_dir = clone_project(config.SHARED_VOLUME_PATH, scan_id, project.repo, project.repo_auth_type,
+                                       scan.branch)
+            scan.source_path = cloned_dir
+            session.add(scan)
+            logger.debug('{} cloned dir {}'.format(scan_id, cloned_dir))
+        except Exception as e:
+            update_scan_state(scan, ScanState.INVALID_BRANCH, session)
+            session.commit()
+
+            logger.error(str(e))
+            raise e
+
+        # if a .deeptracy.yml is found, parse it to a dictionary
         try:
             deeptracy_yml = parse_deeptracy_yml(cloned_dir)
             logger.debug('{} .deeptracy.yml {}'.format(scan_id, 'TRUE' if deeptracy_yml else 'FALSE'))
-        except Exception:
+        except Exception as e:
             update_scan_state(scan, ScanState.INVALID_YML_ON_PROJECT, session)
-            logger.debug('{} unable to parse .deeptracy.yml'.format(scan_id))
-            raise
+            session.commit()
+            logger.error('{} unable to parse .deeptracy.yml'.format(scan_id))
+            raise e
 
         # the language for a scan can be specified on the scan of in the deeptracy file in the sources
         if scan.lang is None:
             if deeptracy_yml is None:
                 update_scan_state(scan, ScanState.CANT_GET_LANGUAGE, session)
+                session.commit()
                 logger.debug('{} unable to retrieve language for scan'.format(scan_id))
                 raise TaskException('unable to retrieve language for scan')
             else:
@@ -71,7 +80,46 @@ def prepare_scan(scan_id: str):
     scan_deps.delay(scan_id)
 
 
-def prepare_path_to_clone_with_local_key(scan_path: str, repo: str, mounted_vol: str, source_folder: str):
+def prepare_path_to_list_branches_with_local_key(scan_path: str, repo: str, mounted_vol: str,
+                                                 mounted_path_branches: str):
+    """
+    Prepare a folder to list a repository branches which needs a local private key.
+
+    LOCAL_PRIVATE_KEY means we need to copy the local private key present in the host to the folder
+    that is being mounted in to the container that is going to perform the actual repo to list branches
+
+    We prepare a script with all the commands needed to perform the branches list, like adding the private key
+    to the container ssh-agent and disabling host-key verification
+
+    :param scan_path: (str) path that we need to prepare
+    :param repo: (str) project repository to clone
+    :param mounted_vol: (str) path to the mounted volume in the container that makes the clone
+    :param mounted_path_branches: (str) name of the path at the file that contains the branches list
+    :return: returns the script to create in the container that makes the branches list
+    """
+
+    logger.debug('prepare repo for list branches with local private key')
+    key_name = 'local_id_rsa'
+    dest_key_path = os.path.join(scan_path, key_name)
+    copyfile(config.LOCAL_PRIVATE_KEY_FILE, dest_key_path)
+
+    # Having the key in the mounted folder, prepare a script to activate the ssh-agent, disable the host
+    # key verification and add the key to it after doing the branches list
+    script_contents = '#!/bin/bash \n' \
+                      'eval `ssh-agent -s` \n' \
+                      'chmod 400 {mounted_vol}/{key} \n' \
+                      'mkdir -p /root/.ssh \n' \
+                      'touch /root/.ssh/config \n' \
+                      'echo -e "StrictHostKeyChecking no" >> /root/.ssh/config \n' \
+                      'ssh-add {mounted_vol}/{key} \n' \
+                      'git  ls-remote --heads {repo} '.format(key=key_name, repo=repo, mounted_vol=mounted_vol)
+
+    script_contents = script_contents + '| awk \'{n=split($2,a,"/"); print a[n]}\' ' \
+        + '>> {mounted_path_branches}\n'.format(mounted_path_branches=mounted_path_branches)
+    return script_contents
+
+
+def prepare_path_to_clone_with_local_key(scan_path: str, repo: str, mounted_vol: str, source_folder: str, branch: str):
     """
     Prepare a folder to clone a repository which needs a local private key.
 
@@ -85,12 +133,11 @@ def prepare_path_to_clone_with_local_key(scan_path: str, repo: str, mounted_vol:
     :param repo: (str) project repository to clone
     :param mounted_vol: (str) path to the mounted volume in the container that makes the clone
     :param source_folder: (str) name of the folder to made the actual clone
+    :param branch: (str) name of the branch to clone
     :return: returns the command to pass to the container that makes the clone
     """
     logger.debug('prepare repo for clone with local private key')
     key_name = 'local_id_rsa'
-    dest_key_path = os.path.join(scan_path, key_name)
-    copyfile(config.LOCAL_PRIVATE_KEY_FILE, dest_key_path)
 
     # Having the key in the mounted folder, preapre a script to activate the ssh-agent, disable the host
     # key verification and add the key to it after doing the clone
@@ -101,10 +148,11 @@ def prepare_path_to_clone_with_local_key(scan_path: str, repo: str, mounted_vol:
                       'touch /root/.ssh/config \n' \
                       'echo -e "StrictHostKeyChecking no" >> /root/.ssh/config \n' \
                       'ssh-add {mounted_vol}/{key} \n' \
-                      'git clone {repo} {mounted_vol}/{source_folder}\n' \
+                      'git clone -b {branch} {repo} {mounted_vol}/{source_folder}\n' \
         .format(
             key=key_name,
             repo=repo,
+            branch=branch,
             mounted_vol=mounted_vol,
             source_folder=source_folder
         )
@@ -119,7 +167,7 @@ def prepare_path_to_clone_with_local_key(scan_path: str, repo: str, mounted_vol:
     return command
 
 
-def clone_project(base_path: str, scan_id: str, repo_url: str, repo_auth_type: str) -> str:
+def clone_project(base_path: str, scan_id: str, repo_url: str, repo_auth_type: str, branch: str = None) -> str:
     """
     Clone a project repository.
 
@@ -132,27 +180,86 @@ def clone_project(base_path: str, scan_id: str, repo_url: str, repo_auth_type: s
     :param scan_id: (str) Scan id that triggers the clone. Its going to be created as a folder inside the base_path
     :param repo_url: (str) project repository url to make the git clone
     :param repo_auth_type: (str) if the repo needs any kind of auth
+    :param branch: (str, Optional) project branch to make the git clone
 
     :return: returns the sources path where the repo is cloned
     """
+
+    branch_file = 'branches.txt'
     source_folder = 'sources'
+
     scan_path = os.path.join(base_path, scan_id)
     scan_path_sources = os.path.join(scan_path, source_folder)
+    scan_path_branches = os.path.join(scan_path, branch_file)
     if not os.path.exists(scan_path):
         os.makedirs(scan_path)
 
     mounted_vol = '/opt/deeptracy'
+    mounted_path_branches = os.path.join(mounted_vol, branch_file)
+
+    if repo_auth_type == RepoAuthType.LOCAL_PRIVATE_KEY.name:
+        # if the project is auth with LOCAL_PRIVATE_KEY prepare the path and get the new command to clone
+        script_contents = prepare_path_to_list_branches_with_local_key(scan_path, repo_url, mounted_vol,
+                                                                       mounted_path_branches)
+    else:
+        # Command to list public repos
+        script_contents = '#!/bin/bash \n' \
+                          + 'git  ls-remote --heads {repo} '.format(repo=repo_url) \
+                          + '| awk \'{n=split($2,a,"/"); print a[n]}\' ' \
+                          + '>> {}'.format(mounted_path_branches)
+
+    # create the script that makes the clone
+    script = os.path.join(scan_path, 'list_branches.sh')
+    with open(script, "w") as f:
+        f.write(script_contents)
+
+    # Command to list public repos
+    command_list_branches = 'bash /opt/deeptracy/list_branches.sh'
+
+    logger.debug('list branches repo with command {}'.format(command_list_branches))
+
+    docker_client = docker.from_env()
+
+    # prepare mounted volumes
+    docker_volumes = {
+        scan_path: {
+            'bind': mounted_vol,
+            'mode': 'rw'
+        }
+    }
+
+    # launch the container with a command that will clone the repo
+    docker_client.containers.run(
+        image='bravissimolabs/alpine-git',
+        command=command_list_branches,
+        remove=True,
+        volumes=docker_volumes
+    )
+
+    branches = open(scan_path_branches, "r").read().splitlines()
+
+    if branch not in branches:
+        raise Exception('The branch {branch} for the scan {scan_id} in the repo {repo} not exits'.format(
+            branch=branch,
+            scan_id=scan_id,
+            repo=repo_url
+        ))
+
+    logger.debug('repo {} branches listed'.format(repo_url))
+
+    # TODO:  Parse the file and verify if the branch exists
 
     # Command to run for public repos
-    command = 'git clone {repo} {mounted_vol}/{source_folder}/'.format(
+    command = 'git clone -b {branch} {repo} {mounted_vol}/{source_folder}/'.format(
+        branch=branch,
         repo=repo_url,
         mounted_vol=mounted_vol,
         source_folder=source_folder
     )
 
     if repo_auth_type == RepoAuthType.LOCAL_PRIVATE_KEY.name:
         # if the project is auth with LOCAL_PRIVATE_KEY prepare the path and get the new command to clone
-        command = prepare_path_to_clone_with_local_key(scan_path, repo_url, mounted_vol, source_folder)
+        command = prepare_path_to_clone_with_local_key(scan_path, repo_url, mounted_vol, source_folder, branch)
 
     logger.debug('clone repo with command {}'.format(command))
 

deeptracy/tasks/scan_deps.py

@@ -40,6 +40,7 @@ def scan_deps(scan_id: str):
 
         # save all dependencies in the database
         add_scan_deps(scan.id, dependencies, datetime.now(), session)
+        scan.total_packages = len(dependencies)
         session.commit()
         logger.debug('saved {} dependencies'.format(len(dependencies)))
 

docker-compose.yml

@@ -8,8 +8,6 @@ services:
       - POSTGRES_PASSWORD=postgres
     ports:
       - 5433:5433
-    environment:
-      - POSTGRES_PASSWORD=postgres
     command: -p 5433
 
   redis:

requirements.txt

@@ -2,8 +2,7 @@ celery==4.1.0
 redis==2.10.6
 psycopg2==2.7.3.1
 pluginbase==0.5
-redis==2.10.6
-deeptracy_core==0.0.24
+deeptracy_core==0.0.27
 requests==2.18.4
 docker==2.5.1
 PyYAML==3.12

tests/acceptance/private_key/local_id_rsa

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEArjlh/B8yFyn2DOQxs7JxA2HgzWq8S55VWSCEQMQSIkawUFcK
+XVT2pVbQbgOR/REBNQGdHREsZMRK4q5o5mJzzw+j2qW6QH2AaoUMNVlm+og5u3HH
+ffG1RDGQHP92MnaO6NRYOApduiJdOeAaHyXGynFwg9M0gNm88lCqva8aRSdw5x2h
+L3/HI9mfU4+J07HbxZW6QWeX+iiQM4Ti3Zh5SqEG0sGg/2PY/Z+9TI0uiYw3GmKP
+DTksA3XmsG8r8xd6kvazKBV58r2WDQpQK8gRt/XN/UDFGaY8p3bMf2pr7uzJhTmc
+T2/Oq23UbTOlYrhJTncglqWp2wgRueTzWNzUruKH8YlICBE9hrSdJQSii1nxYW6u
+O6C1DHkvu0135Utqy3M7W9XEsEX/L+HPRIY8BhFdEb/H70ARH1dYUvDLOeS3g00C
+lZ5ELCCoh9KPdC5cTUANLdtccaazWRrozRTNynxpKhN2mVAKIlQzlExIMnWuzKDX
+8Ow9pRLHPdKfllvwa10impW25fonQQPod3UzmpWa+ZrHc0E4z2qvbfZnbx9a1Qei
+lBtMnlWicIjqXA/l1jUIlK2mbeS0wmEWjmyenuY2EpBDNdoJvbS63VDwVQFDLNym
+tXe6qPYANSkxijUvdeDK4mSjNYuthXHmlnl4ImJSyv/X3IlkgloNpVUlgWcCAwEA
+AQKCAgBQ4y1d27Hq1T16MLvunQshMeEoHcT8Kq+faJxeYWm15jHXqwGR4W99xfeR
+DDFMWAfCb1RdeWquAgBui+ECDXWNDcraKZV2eY4UZl8Lqc1IxdRf9WxUSdMJrSUm
+gWiD901URboLkqx/TZMQ2r3l9ij8Nnyxh25Z4D18Fv2egVxl/PDLLJN4NQ/k0hXJ
+utuXoHzTmDdYRFtn9ks00B5wGWk+v9ImKWWxDM9iuTYvA2E21RWEN3wI/KE1nhiT
+k5nJC/6JmuYE1yu+Ck1PEpm0Kfgn+4Q+b0LTLL3+VgaNpYAk9mR9aQjsTJGcP6qf
+ZJoFYmAOgPptVs6sdWudbfGZMTXAk7K6Q3Fkyg1pnhKlan1Pg0R6CvCf5c9WEWhX
+3Udp872cXRr4pUr0CeDTdIdWEt2OIyNAsEoqPX2aFAJVyHn7gGF4RbpPnin4A677
+sBsoKpDnPFLXs/UPw1ATeIMj4cYx5R+qlWkVuwUdZBEDv9vIjZnFf6EHjphl95rR
+8PuIPIHDuvfWvrY6yUaIICotNe3ucGR329srkTzKPPY/RE9LK+SCDshatqQ+aftb
+gCEK2F6OVIPtxsX8qNog9EOXFoZ4/k8DUBC39+wZ2IlycFkBPYJ4qbU4MK07bRlM
+aknr7Z5JtbzG1X85iA4TN3MpsPc8Q0WD/nOjsGL3k95MzSj8+QKCAQEA4l0HqZ9G
+/Uo+82k4dTCRA+nl71t9KGFR0FvRir/fRKzMkuiQQaDRI3aIH6dLutQPkFHrM7jO
+mv4VGg7nIVPRbIBn1M4fnfUsCJHKX0ZNzkKHzZJx1RLEJ2zzuXTKIwuZRP3SCmp1
+rwhAV1pghpYYgvEasQ1i2ATN0aLTZnDR7fsPBirL/uaMsTFDm4ci6zdSj8zrD11T
+h+thRuRDCOezE9DM29y5KB7hcPUVQoqr4sYUfzz6PpFm69HM076hqsltSxtwV6IX
+Mdfjfpbqanc0V1bDBFQNA9eGVnSMH78iBUlWHHYjemJKHg55t1f2srYyOpUvUm5C
+r3Wb/U5sfei0OwKCAQEAxQjQS4Hhqxtj/riASxIDN0cWIqz4/cGNeOizZ1Yr8e4/
+IhbtITsXHsp40OpSC0sleKS9776SSjnSYonladlkkAjLIGmoN8Y+mfxxyUhBelVo
+jQpao5k2sbuKRO81Tww1lJhRIrQFxtPkA4rQgGFRsX/WbEp7Xs8NWo+8GElx5n/y
+e3hASR1apM0FUa2FPGTtJRZekmOgBOMWRT3vdtq3B216BWdKdqqcm8Ra7yu+Mr/W
+AvIXylRTzm70UggTtghAXXl5iFBqmdt/Upe6cmdyGlb6eRLgIncptGWPLVHPmi+U
+jjxs+zcCGbdIdkLKlWd04AznZDeqSo1HAq484BtwxQKCAQEAtdk4s6rSU99WjSNl
+iLe4eCVqZAzikhMEr9djkgysZ7ZOLmMIqMBx4wRxPMSgUPnVewJkaku4Jsmh2CpK
+wfpdDsZlI49n1PTGKCg/JKUNEnteL+bK7frCfE3Jyp4pWVgTDFrAZz+5RSDi14oy
+a0yfamoZIE15r8LEOfqNzAksjPuYyUKOWTuLoTnLb2FzyvTJrd5YpCI8Xb/ZiSMN
+O1UwBuZB/Qrn8HRGdgFdz/QpO+gXRVSOUL9sqFwGKcFGjTZ+R81GRhBgg9lE2EiT
+D26uLM/1oT+IqSJ2uHOusj5RLWAJ/pllzQazNkw4ufK/rDg0R93bg2QyzFQ2OZvM
+CqZsSwKCAQEAjdX3Y7N+iNbx16ZnLCMuT3eLtrB/mOPg/F2+7693ePPBXL/WOaq4
+zQCBkhprrxeMRNmKpO+xjV8sVKThkZ3dp1W9K3sDjgrWe7DfFD9Aa1jaJ1WBw+0C
+E1VmhpMP8/RdCbfQCERBOEzGAcrGpvPng/f7mf7P6oLwIPYBOBZ+uKf4HOuGk1tf
+Ke0wXAimNcCNebotoAG5amsyV+vq2ss9IEqtoQAm+V991x+1OiBqDUxNOdeeSpcD
+sUHr/LU1wz8TXnaYhwkkg9cfQ4xXug0/dTiRm+B7mjPfwK2vMLeI03xW20EiToDd
+sTdgIUmZXY2Y88Q2Oc1cd8hVitd0mPsH5QKCAQEAy7wwQbhDQtSUSFufYRicoMl1
+YjmwVEGnojPFzZJGfuHV40d1OJZ1YjmQ8qPSY+lrgGrjNyzcdNatZTWKrqldUFfH
+En/JX66FHyrfJB8+SUJgmvzVlLeChQqgQaWDSC/h1YOCM9pANd+VAPmc+SKBoeT3
+t+khIwA15q44JLs/Dk6ff3byzONSN9P1oPYoPQwjctO5PPd/BlPj4sgUOfkgeGnV
+9/k/+r1CGxYPqL6aQdEfFpkXqmitY9/SVeDDqDGCqaYNDHxh/qpFVqqfhWZF23av
+nQzQaJKFOKD9hipk3gFbMKm4fXddMryZbPeK3ZvHQddFTdGWCdo5SDaH0G0ebw==
+-----END RSA PRIVATE KEY-----

tests/acceptance/steps/database.py

@@ -63,16 +63,17 @@ def step_impl(context, plugin_name, lang):
 @given(u'a project with "{repo}" repo exists in the database')
 def step_impl(context, repo):
     project_id = uuid.uuid4()
-    sql = text('INSERT INTO project (id, repo) VALUES (:id, :repo)')
-    context.engine.execute(sql, id=project_id, repo=repo)
+    sql = text('INSERT INTO project (id, name, repo) VALUES (:id, :name, :repo)')
+    context.engine.execute(sql, id=project_id, name='test', repo=repo)
     context.project_id = project_id
 
 
 @given(u'a scan for lang "{lang}" exists for the project')
 def step_impl(context, lang):
     scan_id = '123'
-    sql = text('INSERT INTO scan (id, project_id, lang, created) VALUES (:id, :project_id, :lang, :created)')
-    context.engine.execute(sql, id=scan_id, project_id=context.project_id, lang=lang, created=datetime.now())
+    sql = text('INSERT INTO scan (id, project_id, lang, branch, created) '
+               'VALUES (:id, :project_id, :lang, :branch, :created)')
+    context.engine.execute(sql, id=scan_id, project_id=context.project_id, lang=lang, branch='master', created=datetime.now())
     context.scan_id = scan_id
 
 
@@ -98,11 +99,12 @@ def step_impl(context):
 def step_impl(context, repo, repo_auth_type):
     project_id = '123'
 
-    sql = text('INSERT INTO project (id, repo, repo_auth_type) '
-               'VALUES (:id, :repo, :repo_auth_type)')
+    sql = text('INSERT INTO project (id, name, repo, repo_auth_type) '
+               'VALUES (:id, :name, :repo, :repo_auth_type)')
 
     context.engine.execute(sql,
                            id=project_id,
+                           name='test',
                            repo=repo,
                            repo_auth_type=repo_auth_type)