Dev

.dockerignore

@@ -6,4 +6,8 @@
 .gitignore
 README.md
 .dockerignore
-.git
+.git
+
+/backend/data/setup_key
+/backend/data/keys*
+/backend/data/sboms*

.gitignore

@@ -1,7 +1,6 @@
 
-*.pyc
-*.pyc
 *.pyc
 backend/.idea
-*.pyc
-*.pyc
+backend/data/sboms
+backend/data/db.sqlite3
+backend/data/keys 

.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+    "sqltools.connections": [
+        {
+            "previewLimit": 50,
+            "driver": "SQLite",
+            "name": "db",
+            "database": "${workspaceFolder:dependify}/backend/data/db.sqlite3"
+        }
+    ],
+    "sqltools.useNodeRuntime": true
+}

Dockerfile

@@ -52,7 +52,9 @@ COPY ./backend/requirements.txt ./
 # Copy the Django source code
 COPY ./backend .
 
+RUN apt-get update && apt-get install -y openssh-client
 
+RUN apt-get update && apt-get install -y git
 
 RUN pip3 install -r requirements.txt
 
@@ -67,6 +69,8 @@ COPY start_up.sh ./start_up.sh
 
 RUN chmod +x ./start_up.sh
 
+RUN python manage.py flush --no-input
+
 
 # Start the Django server
 #CMD ["python3", "manage.py", "runserver", "0.0.0.0:8000" ,";","npm", "run", "start"]

backend/backend/urls.py

@@ -23,15 +23,25 @@
 from playground.views import Login
 from playground.views import Logout
 from playground.views import User
-
+from playground.views import IsSetup
+from playground.views import Generate_datasource
+from playground.views import Confirm_datasource
+from playground.views import Get_datasource
+from playground.views import Del_datasource
 
 urlpatterns = [
     #path('admin/', admin.site.urls),
     path ('api/scan/', Scan.as_view()),
-    path ('api/setup', Setup.as_view()),
+    path ('api/setup/', Setup.as_view()),
     path ('api/setup_registration/', SetupUserRegistration.as_view()),
     path ('api/registration/',UserRegistration.as_view()),
     path ('api/login/',Login.as_view()),
     path ('api/logout/',Logout.as_view()),
     path ('api/user/',User.as_view()),
+    path ('api/is_setup/',IsSetup.as_view()),
+    path ('api/generate_datasource/',Generate_datasource.as_view()),
+    path ('api/confirm_datasource/',Confirm_datasource.as_view()),
+    path ('api/get_datasource/',Get_datasource.as_view()),
+    path ('api/del_datasource/',Del_datasource.as_view()),
+
 ]

backend/data/setup_key

@@ -0,0 +1 @@
+lucaslucaslucas

backend/functions/datasource.py

@@ -0,0 +1,22 @@
+from git import Repo
+import paramiko
+from playground.models import newDataSource as DataSource
+from django.apps import AppConfig
+
+class FunctionsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'functions'
+
+    def ready(self):
+        clone_datasource('pw-demo-sboms')
+def clone_datasource(name):
+    repo_path = '/data/sboms/'
+    datasource = DataSource.objects.get(name=name)
+    repo_url = datasource.url
+    key = datasource.key
+    repo = Repo.clone_from(repo_url, repo_path, key=key)
+
+
+def fetch_datasource():
+    pass
+

backend/functions/scan.py

@@ -1,6 +1,7 @@
 import os
 import json
-
+import glob
+import math
 import numpy as np
 import pandas as pd
 import functions.version as version_parser
@@ -10,33 +11,42 @@ def find_dependencies_in_sboms(name: str, version: [str], exactMatch: bool) -> o
     # output = {'label': source, 'name': source.lower(), 'type': source.lower(), 'results': []}
     output = []
     print("Searching for dependencies in SBOMs")
-    path = './functions/sboms'
-    for file in os.listdir(path):
+    path = './data/sboms'
+    # search for each file in the folder and subfolders
+
+    for file in glob.glob(path + '/**/*.json', recursive=True):
         if file.endswith(".json"):
-            with open(path + '/' + file, encoding="utf-8") as json_file:
+            with open(file, encoding="utf-8") as json_file:
                 data = json.load(json_file)
+                data_type = ''
                 try:
-                    type = data['bomFormat']
+                    data_type = data['bomFormat']
                     image = data['metadata']['component']['name']
                 except:
                     1 + 1
                 try:
-                    type = data['spdxVersion']
+                    data_type = data['spdxVersion']
                     image = data['name']
                 except:
                     1 + 1
-                if type == 'CycloneDX':
+                if data_type == 'CycloneDX':
                     components = data['components']
                     df = pd.json_normalize(components)
-                    cols = ['name', 'version']
+                    cols = ['name', 'version', 'purl']
                     df = df[cols]
                     df = df.rename(
                         columns={'name': 'label', 'id': 'license_id', 'url': 'reference_url'})
-                elif str(type).find('SPDX') != -1:
+                elif str(data_type).find('SPDX') != -1:
                     packages = data['packages']
                     df = pd.json_normalize(packages)
                     cols = ['name', 'versionInfo']
+                    externalRefs = pd.json_normalize(packages, record_path='externalRefs')
+                    # get all where referenceType is 'PACKAGE-MANAGER'
+                    purl = externalRefs[externalRefs['referenceType'] == 'purl']
+                    purl = purl.reset_index(drop=True)
                     df = df[cols]
+                    # add the reference url from purl to df
+                    df['purl'] = purl['referenceLocator']
                     df = df.rename(
                         columns={'name': 'label', 'versionInfo': 'version'})
                 else:
@@ -50,17 +60,23 @@ def find_dependencies_in_sboms(name: str, version: [str], exactMatch: bool) -> o
                 df = check_versions(df, version)
                 if not df.empty:
                     results = df.to_dict(orient='index')
+                    for v in results.values():
+                        try:
+                            if math.isnan(v['purl']):
+                                v['purl'] = "[!] page url not found"
+                        except:
+                            pass
                     for v in results.values():
                         v['sbomFile'] = file
                         v['dockerImage'] = image
                         projectName = v['dockerImage'].split(':')[0]
                         dockerVersion = v['dockerImage'].split(':')[1]
                     temp = {'name': projectName, 'version': dockerVersion, 'dockerImage': v['dockerImage'],
                             'sbomFile': v['sbomFile'],
-                            'results': [{'label': v['label'], 'version': v['version']} for v in
+                            'results': [{'label': v['label'], 'version': v['version'], 'purl': v['purl']} for v in
                                         results.values()]}
                     output.append(temp)
-
+    #print(json.dumps(output, indent=2))
     return output
 
 

backend/functions/version.py

@@ -12,9 +12,22 @@ def __init__(self, mayor_, minor_ = "none", revision_ = "none"):
       self.revision = 0
     else:
       txt = mayor_.split('.')
-      self.mayor = int(self.pruned(txt[0]))
-      self.minor = int(self.pruned(txt[1]))
-      self.revision = int(self.pruned(txt[2]))
+      if len(txt) >= 3:
+        self.mayor = int(self.pruned(txt[0]))
+        self.minor = int(self.pruned(txt[1]))
+        self.revision = int(self.pruned(txt[2]))
+      elif len(txt) == 2:
+        self.mayor = int(self.pruned(txt[0]))
+        self.minor = int(self.pruned(txt[1]))
+        self.revision = 0
+      elif len(txt) == 1:
+        self.mayor = int(self.pruned(txt[0]))
+        self.minor = 0
+        self.revision = 0
+      else:
+        self.mayor = 0
+        self.minor = 0
+        self.revision = 0
 
   def pruned(self, txt):
     for character in list(string.ascii_letters + string.punctuation):

backend/playground/apps.py

@@ -1,6 +1,42 @@
+import os
+
 from django.apps import AppConfig
 
+def toUrl(text):
+    return text.replace(" ", "_")
 
 class PlaygroundConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'playground'
+
+    def ready(self):
+        import os
+        import subprocess
+        from playground.models import newDataSource as DataSource
+
+        def clone_datasource(name):
+
+            if not os.path.exists('./data/sboms'):
+                os.makedirs('./data/sboms', exist_ok=True)
+
+            repo_path = f'./data/sboms/{name}'
+            current_datasource = DataSource.objects.get(name=name)
+            repo_url = current_datasource.url
+            absolute_key_path = os.path.abspath(f'./data/keys/{toUrl(name)}_private_key')
+            absolute_key_path = absolute_key_path.replace('\\', '/')
+
+
+            if not os.path.exists(repo_path):
+                subprocess.run(['git', 'config', '--global', 'core.sshCommand',
+                                f'ssh -i {absolute_key_path}  -F /dev/null'])
+                subprocess.run(['git', 'clone', repo_url, repo_path])
+            else:
+                subprocess.run(['git', 'config', '--global', 'core.sshCommand',
+                                f'ssh -i {absolute_key_path} -F /dev/null'], cwd=repo_path)
+                subprocess.run(['git', 'pull'], cwd=repo_path)
+
+
+
+
+        for datasource in DataSource.objects.all():
+            clone_datasource(datasource.name)

backend/playground/migrations/0003_datasource.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2 on 2023-06-06 12:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('playground', '0002_registrationkey'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DataSource',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=30)),
+                ('description', models.CharField(max_length=30)),
+                ('url', models.CharField(max_length=30)),
+                ('key', models.TextField()),
+            ],
+        ),
+    ]

backend/playground/migrations/0004_newdatasource.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2 on 2023-06-15 13:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('playground', '0003_datasource'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NewDataSource',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=30)),
+                ('description', models.CharField(max_length=30)),
+                ('url', models.CharField(max_length=30)),
+                ('key', models.TextField()),
+                ('status', models.BooleanField(default=False)),
+                ('lastSync', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+    ]

backend/playground/migrations/0005_delete_datasource.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.2 on 2023-06-15 13:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('playground', '0004_newdatasource'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='DataSource',
+        ),
+    ]

backend/playground/models.py

@@ -38,3 +38,11 @@ class RegistrationKey(models.Model):
     def __str__(self):
         return self.key
 
+
+class newDataSource(models.Model):
+    name = models.CharField(max_length=30)
+    description = models.CharField(max_length=30)
+    url = models.CharField(max_length=30)
+    key = models.TextField()
+    status = models.BooleanField(default=False)
+    lastSync = models.DateTimeField(auto_now=True)