merge 1.5.4rc6

.dependabot/config.yml

@@ -14,4 +14,3 @@ update_configs:
     directory: "/"
     target_branch: "dev"
     update_schedule: "weekly"
-

.github/release-drafter.yml

@@ -1,4 +1,24 @@
+name-template: 'v$NEXT_PATCH_VERSION 🌈'
+tag-template: 'v$NEXT_PATCH_VERSION'
+categories:
+  - title: '🚀 New scanners'
+    labels:
+      - 'Import Scans'
+  - title: '🚀 Features and enhancements'
+    labels:
+      - 'feature'
+      - 'enhancement'
+  - title: '🐛 Bug Fixes'
+    labels:
+      - 'fix'
+      - 'bugfix'
+      - 'bug'
+  - title: '🧰 Maintenance'
+    labels: 
+      - 'dependencies'
+      - 'maintenance'
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
 template: |
-  ## What's Changed
+  ## Changes
 
   $CHANGES

.github/workflows/release-drafter.yml

@@ -0,0 +1,19 @@
+name: Release Drafter
+
+on:
+  push:
+    # branches to consider in the event; optional, defaults to all
+    branches:
+      - master
+
+jobs:
+  update_release_draft:
+    runs-on: ubuntu-latest
+    steps:
+      # Drafts your next Release notes as Pull Requests are merged into "master"
+      - uses: release-drafter/release-drafter@v5.6.1
+        # with:
+          # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml
+          # config-name: my-config.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -107,3 +107,8 @@ quick.bash
 
 #visual studio code
 *.code-workspace
+
+# pipenv
+Pipfile
+Pipfile*
+

.travis.yml

@@ -16,6 +16,7 @@ env:
     - TEST=flake8
     - TEST=snyk
     - TEST=docker
+    - TEST=integration_tests
 matrix: 
   allow_failures:
     - env: TEST=snyk

DOCKER.md

@@ -1,17 +1,17 @@
-# Run with Docker Compose
+# Running with Docker Compose
 
 Docker compose is not intended for production use.
 If you want to deploy a containerized DefectDojo to a production environment,
 use the [Default installation](setup/README.md) approach.
 
-## Prerequisites
+# Prerequisites
 *  Docker version
     *  Installing with docker-compose requires at least docker 18.09.4 and docker-compose 1.24.0. See "Checking Docker versions" below for version errors during running docker-compose.
 *  Proxies
     *  If you're behind a corporate proxy check https://docs.docker.com/network/proxy/ . 
 
 
-## Setup via Docker Compose - introduction
+# Setup via Docker Compose - introduction
 
 DefectDojo needs several docker images to run. Two of them depend on DefectDojo code:
 
@@ -28,8 +28,8 @@ When running the application without building images, the application will run b
     *  https://hub.docker.com/r/defectdojo/defectdojo-nginx
 
 
-## Setup via Docker Compose - building and running the application
-### Building images
+# Setup via Docker Compose - building and running the application
+## Building images
 
 To build images and put them in your local docker cache, run:
 
@@ -40,7 +40,7 @@ docker-compose build
 To build a single image, run: 
 
 ```zsh
-docker-compose build django
+docker-compose build uwsgi
 ```
 or
 
@@ -49,7 +49,7 @@ docker-compose build nginx
 ```
 
 
-### Run with Docker compose in release mode
+## Run with Docker compose in release mode
 To run the application based on previously built image (or based on dockerhub images if none was locally built), run: 
 
 ```zsh
@@ -62,7 +62,7 @@ This will run the application based on docker-compose.yml only.
 In this setup, you need to rebuild django and/or nginx images after each code change and restart the containers. 
 
 
-### Run with Docker compose in development mode with hot-reloading
+## Run with Docker compose in development mode with hot-reloading
 
 For development, use: 
 
@@ -82,7 +82,7 @@ This will run the application based on merged configurations from docker-compose
 * Hot-reloading for the **celeryworker** container is not yet implemented. When working on deduplication for example, restart the celeryworker container with: 
 
 ```
-docker restart django-defectdojo_celeryworker_1
+docker-compose restart celeryworker
 ```
 
 *  The mysql port is forwarded to the host so that you can access your database from outside the container. 
@@ -100,32 +100,66 @@ To update changes in static resources, served by nginx, just refresh the browser
 id -u
 ```
 
-### Access the application
-Navigate to <http://localhost:8080> where you can log in with username admin.
-To find out the admin password, check the very beginning of the console
-output of the initializer container, typically name 'django-defectdojo_initializer_1', or run the following:
+## Run with Docker compose in development mode with ptvsd (remote debug)
+
+If you want to be able to step in your code, you can activate ptvsd.Server.
+
+You can launch your local dev instance of DefectDojo as
 
 ```zsh
-container_id=(`docker ps -a \
---filter "name=django-defectdojo_initializer_1" \
-| awk 'FNR == 2 {print $1}'`) && \
-docker logs $container_id 2>&1 | grep "Admin password:"
+cp dojo/settings/settings.dist.py dojo/settings/settings.py
+docker/setEnv.sh ptvsd
+docker-compose up
 ```
 
-or:
+This will run the application based on merged configurations from docker-compose.yml and docker-compose.override.ptvsd.yml.
+
+The default configuration assumes port 3000 by default for ptvsd, and you should access the DefectDojo UI on port 8000 instead of port 8080, as the uwsgi container will serve directly.
+
+### VS code
+Add the following python debug configuration (You would have to install the `ms-python.python`. Other setup may work.)
+
+```
+  {
+      "name": "Remote DefectDojo",
+      "type": "python",
+      "request": "attach",
+      "pathMappings": [
+          {
+              "localRoot": "${workspaceFolder}",
+              "remoteRoot": "/app"
+          }
+      ],
+      "port": 3000,
+      "host": "localhost"
+  }
+```
+
+You can now launch the remote debug from VS Code, place your breakpoints and step through the code.
+
+> At present, 2 caveats:
+> - Static will not be present. You would have to `docker cp` them over from the nginx container
+> - For some reason, the page loading may hang. You can stop the loading and reload, the page will ultimately appear.
+
+
+## Access the application
+Navigate to <http://localhost:8080> where you can log in with username admin.
+To find out the admin password, check the very beginning of the console
+output of the initializer container by running:
 
 ```zsh
-docker logs django-defectdojo_initializer_1
+docker-compose logs initializer | grep "Admin password:"
 ```
 
 Make sure you write down the first password generated as you'll need it when re-starting the application.
 
-### Disable the database initialization
+# Exploitation, versioning
+## Disable the database initialization
 The initializer container can be disabled by exporting: `export DD_INITIALIZE=false`. 
 
 This will ensure that the database remains unchanged when re-running the application, keeping your previous settings and admin password.
 
-### Versioning
+## Versioning
 In order to use a specific version when building the images and running the containers, set the environment with 
 *  For the nginx image: `NGINX_VERSION=x.y.z`
 *  For the django image: `DJANGO_VERSION=x.y.z`
@@ -149,9 +183,7 @@ aedc404d6dee        defectdojo/defectdojo-nginx:1.0.0     "/entrypoint-nginx.sh"
 ```
 
 
-
-
-### Clean up Docker Compose
+## Clean up Docker Compose
 
 Removes all containers
 
@@ -165,13 +197,52 @@ Removes all containers, networks and the database volume
 docker-compose down --volumes
 ```
 
-### Run the unit-tests with docker
-#### Introduction
+# Run with docker using https
+To secure the application by https, follow those steps
+*  Generate a private key without password
+*  Generate a CSR (Certificate Signing Request)
+*  Have the CSR signed by a certificate authority
+*  Place the private key and the certificate under the nginx folder
+*  Replace nginx/nginx.conf by nginx/nginx_TLS.conf
+*  In nginx.conf, update that part: 
+```
+        server_name                 your.servername.com;
+        ssl_certificate             /yourCertificate.cer;
+        ssl_certificate_key         /yourPrivateKey.key;
+```
+* Protect your private key from other users: 
+```
+chmod 400 nginx/*.key
+```
+* Rebuild the nginx image in order to place the private key and the certificate where nginx will find them (under / in the nginx container):
+
+```docker build  -t defectdojo/defectdojo-nginx -f Dockerfile.nginx .```
+
+
+* Run defectDojo with: 
+```
+rm -f docker-compose.override.yml
+ln -s docker-compose.override.https.yml docker-compose.override.yml
+docker-compose up
+```
+
+The default https port is 8083.
+
+To change the port:
+- update `nginx.conf`
+- update `docker-compose.override.https.yml` or set DD_PORT in the environment)
+- restart the application
+
+NB: some third party software may require to change the exposed port in Dockerfile.nginx as they use docker-compose declarations to discover which ports to map when publishing the application.
+
+
+# Run the unit-tests with docker
+## Introduction
 The unit-tests are under `dojo/unittests`
 
 
 
-#### Running the unit-tests 
+## Running the unit-tests 
 This will run all the tests and leave the uwsgi container up: 
 
 ```
@@ -182,7 +253,7 @@ docker-compose up
 Enter the container to run more tests:
 
 ```
-docker exec -it django-defectdojo_uwsgi_1 bash
+docker-compose exec uwsgi bash
 ```
 Rerun all the tests:
 
@@ -202,7 +273,7 @@ Run a single test. Example:
 python manage.py test dojo.unittests.test_dependency_check_parser.TestDependencyCheckParser.test_parse_without_file_has_no_findings --keepdb
 ```
 
-## Checking Docker versions
+# Checking Docker versions
 
 Run the following to determine the versions for docker and docker-compose:
 

Dockerfile.busybox

@@ -1,2 +1,2 @@
-FROM busybox:1.31.0-musl
+FROM busybox:1.31.1-musl
 ENTRYPOINT ["/bin/echo", "hello world"]

Dockerfile.django

@@ -4,10 +4,10 @@
 # The code for the build image should be idendical with the code in
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
-# Using 3.5.7 to avoid compatibility issues that may be introduced by python 3.5.6 and 3.5.7. 
+# Using 3.5.7 to avoid compatibility issues that may be introduced by python 3.6 and 3.7.
 # Please upgrade before end-of-life in september 2020!
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.5.7-buster@sha256:4598d4365bb7a8628ba840f87406323e699c4da01ae6f926ff33787c63230779 as build
+FROM python:3.5.9-buster@sha256:1baef6be00b82fbd77f1b60ab227a1dbede6f23825ce1b7f1e9c6f7d1469a45c as build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -24,7 +24,7 @@ RUN \
 COPY requirements.txt ./
 RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
-FROM python:3.5.7-slim-buster@sha256:127fee645393d311c7fbc5e8c2e5034f10a4e66b47c9273d4dbe5da2926fc3f2
+FROM python:3.5.9-slim-buster@sha256:dfb042910e4ef352b5c6aa223031ce768f53f4f1aacf95936152e5508162bcb0
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -60,6 +60,7 @@ COPY \
   docker/entrypoint-initializer.sh \
   docker/entrypoint-uwsgi.sh \
   docker/entrypoint-uwsgi-dev.sh \
+  docker/entrypoint-uwsgi-ptvsd.sh \
   docker/entrypoint-unit-tests.sh \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
@@ -76,7 +77,7 @@ RUN \
   chmod g=u /var/run && \
   true
 USER root
-RUN chmod 0777 /app
+RUN chmod -R 0777 /app
 USER 1001
 ENV \
   DD_ADMIN_USER=admin \
@@ -99,11 +100,9 @@ ENV \
   DD_DATABASE_PASSWORD="defectdojo" \
   DD_DATABASE_PORT="3306" \
   DD_DATABASE_USER="defectdojo" \
-  DD_SECRET_KEY="hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq" \
-  DD_CREDENTIAL_AES_256_KEY="&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw" \
   DD_INITIALIZE=true \
   DD_UWSGI_MODE="socket" \
   DD_UWSGI_ENDPOINT="0.0.0.0:3031" \
-  DD_DJANGO_ADMIN_ENABLED="on" \
-  DD_TRACK_MIGRATIONS="on"
+  DD_DJANGO_ADMIN_ENABLED="True" \
+  DD_TRACK_MIGRATIONS="True"
 ENTRYPOINT ["/entrypoint-uwsgi.sh"]

Dockerfile.nginx

@@ -3,7 +3,7 @@
 # The code for the build image should be idendical with the code in
 # Dockerfile.django to use the caching mechanism of Docker.
 
-FROM python:3.5.7-buster@sha256:4598d4365bb7a8628ba840f87406323e699c4da01ae6f926ff33787c63230779 as build
+FROM python:3.5.9-buster@sha256:1baef6be00b82fbd77f1b60ab227a1dbede6f23825ce1b7f1e9c6f7d1469a45c as build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -55,13 +55,14 @@ RUN \
   python3 manage.py collectstatic --noinput && \
   true
 
-FROM nginx:1.17.2@sha256:eb3320e2f9ca409b7c0aa71aea3cf7ce7d018f03a372564dbdb023646958770b
+FROM nginx:1.17.7@sha256:89a42c3ba15f09a3fbe39856bddacdf9e94cd03df7403cad4fc105088e268fc9
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/
 COPY wsgi_params nginx/nginx.conf /etc/nginx/
-COPY docker/entrypoint-nginx.sh /
+COPY docker/entrypoint-nginx.sh nginx/*.cer nginx/*.key /
 RUN \
   chmod -R g=u /var/cache/nginx && \
   chmod -R g=u /var/run && \
+  if [ -f /*.key -o -f /*.cer ]; then chown 1001 /*.key /*.cer; fi && \
   true
 ENV \
   DD_UWSGI_PASS="uwsgi_server" \