Merge pull request #67 from BBVA/develop

Add Java lang

deeptracy/tasks/get_vulnerabilities.py

@@ -35,24 +35,7 @@ def get_vulnerabilities(scan_id: str):
     with db.session_scope() as session:
         logger.debug('{} extract dependencies'.format(scan_id))
 
-        scan_deps = []
-
-        def format(raw_dep):
-
-            parts = raw_dep.split(':')
-            if len(parts) == 3:
-                library_parts = parts[1].split('@')
-
-                if len(library_parts) > 2:
-                    name_package = '@'.join(library_parts[:-1])
-                else:
-                    name_package = library_parts[0]
-
-                version_part = library_parts[-1]
-                scan_deps.append([name_package, version_part])
-
-        scans_deps_aux = get_scan_deps(scan_id, session)
-        [format(scan.raw_dep) for scan in scans_deps_aux]
+        scan_deps = get_scan_deps(scan_id, session)
         scan_deps_len = len(scan_deps)
 
         scan = get_scan(scan_id, session)
@@ -61,10 +44,10 @@ def format(raw_dep):
         total_vulnerabilities = []
 
         def get_response(i, scan_dep):
-            [package, version] = scan_dep
+            [package, version] = scan_dep.raw_dep.split(':')
             url = '{}/batch'.format(PATTON_URI)
 
-            response = requests.post(url, json=[scan_dep]).json()
+            response = requests.post(url, json=[[package, version]]).json()
             print(response)
             logger.info("Procesado {} de {}".format(i, scan_deps_len))
 

deeptracy/tasks/scan_deps.py

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import re
+from os import listdir
+from os.path import join
 
 from datetime import datetime
 
@@ -78,6 +81,9 @@ def get_dependencies(lang: str, sources: str):
     if lang == 'nodejs':
         return get_dependencies_for_nodejs(sources, mounted_vol, docker_volumes)
 
+    if lang == 'java':
+        return get_dependencies_for_java(sources, mounted_vol, docker_volumes)
+
 
 def get_dependencies_for_nodejs(sources: str, mounted_vol: str, docker_volumes: dict):
     image = 'node:latest'
@@ -87,9 +93,7 @@ def get_dependencies_for_nodejs(sources: str, mounted_vol: str, docker_volumes:
                       'cd {mounted_vol} \n' \
                       'npm install --ignore-scripts \n' \
                       'npm ls --parseable --long' \
-        .format(
-            mounted_vol=mounted_vol
-        )
+        .format(mounted_vol=mounted_vol)
 
     # create the script that makes the clone
     script = os.path.join(sources, 'get_deps.sh')
@@ -119,6 +123,74 @@ def get_dependencies_for_nodejs(sources: str, mounted_vol: str, docker_volumes:
             # TODO: deps has paths and need to be parsed carefully
             dep_split = line.split('node_modules/', 1)
             # parsed_dep_list.append(dep_split[-1])
-            dep_list.append(dep_split[1])
+            parts = dep_split[1].split(':')
+            if len(parts) == 3:
+                library_parts = parts[1].split('@')
+
+                if len(library_parts) > 2:
+                    name_package = '@'.join(library_parts[:-1])
+                else:
+                    name_package = library_parts[0]
+
+                version_part = library_parts[-1]
+                dep_list.append('{}:{}'.format(name_package, version_part))
+
+    return dep_list
+
+
+def get_dependencies_for_java(sources: str, mounted_vol: str, docker_volumes: dict):
+    image = 'maven-gradle:0.0.1'
+    script_contents = ('#!/bin/bash \n'
+                       'mkdir /tmp/deeptracy \n'
+                       'cp -R {mounted_vol} /tmp/deeptracy \n'
+                       'cd {mounted_vol} \n'
+                       'COUNT_GRADLE=$(find -name gradle | wc -l) \n'
+                       'if [ $COUNT_GRADLE -gt 0 ] ; then \n'
+                       ' gradle dependencies  --configuration compile > gradle.txt \n'
+                       'else \n'
+                       ' mvn dependency:tree -DoutputFile=maven.txt \n'
+                       'fi \n').format(
+        mounted_vol=mounted_vol
+    )
+
+    # create the script that makes the clone
+    script = os.path.join(sources, 'get_deps.sh')
+    with open(script, "w") as f:
+        f.write(script_contents)
 
+    os.system('chmod +x {}'.format(script))
+    command = os.path.join(mounted_vol, 'get_deps.sh')  # execute script IGNORING errors
+
+    logger.debug('extract deps with command {}'.format(command))
+
+    docker_client = docker.from_env()
+
+    docker_client.containers.run(
+        image=image,
+        command=command,
+        remove=True,
+        volumes=docker_volumes,
+        detach=False
+    )
+
+    dep_list = []
+    if "gradle.txt" in listdir(sources):
+        file = open(join(sources, "gradle.txt"), 'r')
+        for line in file.readlines():
+            if '\--- ' in line or '+--- ' in line:
+                if '\--- ' in line:
+                    pattern = re.compile(r'[A-Z]*\--- ')
+                if '+--- ' in line:
+                    pattern = re.compile(r'[A-Z]*\+--- ')
+                [package, name_package, version_part] = pattern.split(line)[1].replace("\n", "").split(":")
+                dep_list.append('{}:{}'.format(name_package, version_part))
+
+    if "maven.txt" in listdir(sources):
+        file = open(join(sources, "maven.txt"), 'r')
+        for line in file.readlines():
+            if '+- ' in line or '\- ' in line:
+                pattern = re.compile(r'[+-\\ \|]* ([\w:.-]*)')
+                [package, name_package, type, version_part, extra] = pattern \
+                    .split(line)[1].replace("\n", "").split(":")
+                dep_list.append('{}:{}'.format(name_package, version_part))
     return dep_list

docker/images/maven-gradle/Dockerfile

@@ -0,0 +1,38 @@
+FROM maven:latest
+
+CMD ["gradle"]
+
+ENV GRADLE_HOME /opt/gradle
+ENV GRADLE_VERSION 4.4.1
+
+ARG GRADLE_DOWNLOAD_SHA256=e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389
+RUN set -o errexit -o nounset \
+	&& echo "Downloading Gradle" \
+	&& wget --no-verbose --output-document=gradle.zip "https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" \
+	\
+	&& echo "Checking download hash" \
+	&& echo "${GRADLE_DOWNLOAD_SHA256} *gradle.zip" | sha256sum --check - \
+	\
+	&& echo "Installing Gradle" \
+	&& unzip gradle.zip \
+	&& rm gradle.zip \
+	&& mv "gradle-${GRADLE_VERSION}" "${GRADLE_HOME}/" \
+	&& ln --symbolic "${GRADLE_HOME}/bin/gradle" /usr/bin/gradle \
+	\
+	&& echo "Adding gradle user and group" \
+	&& groupadd --system --gid 1000 gradle \
+	&& useradd --system --gid gradle --uid 1000 --shell /bin/bash --create-home gradle \
+	&& mkdir /home/gradle/.gradle \
+	&& chown --recursive gradle:gradle /home/gradle \
+	\
+	&& echo "Symlinking root Gradle cache to gradle Gradle cache" \
+	&& ln -s /home/gradle/.gradle /root/.gradle
+
+# Create Gradle volume
+USER gradle
+VOLUME "/home/gradle/.gradle"
+WORKDIR /home/gradle
+
+RUN set -o errexit -o nounset \
+	&& echo "Testing Gradle installation" \
+	&& gradle --version