Skip to content
Permalink
Browse files

Docker & Updates (#764)

* Settings.py environment files and migrations support

* Moved settings to .env and created more container friendly services

* Travis improvements

* Added mobsf importer

* Travis dockerfile

* branch check

* Version bump
  • Loading branch information...
aaronweaver committed Oct 5, 2018
1 parent 511084c commit 4f66fb7ae65f5227ba6bbcac3019232cb309ac13
Showing with 43,582 additions and 521 deletions.
  1. +4 −0 .dockerignore
  2. +2 −0 .gitignore
  3. +12 −27 .travis.yml
  4. +20 −45 Dockerfile
  5. +3 −2 PULL_REQUEST_TEMPLATE.md
  6. +2 −2 README.md
  7. +0 −10 batch_mode.properties
  8. +1 −1 components/package.json
  9. +1 −1 dojo/__init__.py
  10. +4 −0 dojo/admin.py
  11. +3 −4 dojo/api_v2/serializers.py
  12. +1 −1 dojo/api_v2/views.py
  13. +1,319 −0 dojo/db_migrations/0001_initial.py
  14. +20 −0 dojo/db_migrations/0002_test_description.py
  15. +35 −0 dojo/db_migrations/0003_auto_20181004_1524.py
  16. 0 dojo/db_migrations/__init__.py
  17. +4 −3 dojo/filters.py
  18. +41,058 −0 dojo/fixtures/defect_dojo_sample_data.json
  19. +1 −1 dojo/fixtures/dojo_testdata.json
  20. +4 −2 dojo/forms.py
  21. +1 −1 dojo/home/views.py
  22. +9 −4 dojo/models.py
  23. +229 −105 dojo/settings/settings.dist.py
  24. +71 −0 dojo/settings/template-env
  25. +2 −2 dojo/tasks.py
  26. +38 −18 dojo/templates/base.html
  27. +1 −1 dojo/templates/dojo/delete_product.html
  28. +1 −1 dojo/templates/dojo/delete_rule.html
  29. +2 −2 dojo/templates/dojo/endpoints.html
  30. +1 −0 dojo/templates/dojo/import_scan_results.html
  31. +8 −6 dojo/templates/dojo/snippets/engagement_list.html
  32. +6 −2 dojo/templates/dojo/view_endpoint.html
  33. +14 −5 dojo/templates/dojo/view_eng.html
  34. +35 −25 dojo/templates/dojo/view_finding.html
  35. +1 −1 dojo/templates/dojo/view_product_details.html
  36. +16 −0 dojo/templates/dojo/view_test.html
  37. +16 −0 dojo/templatetags/display_tags.py
  38. +8 −33 dojo/tool_config/views.py
  39. +22 −17 dojo/tools/burp/parser.py
  40. +3 −3 dojo/tools/factory.py
  41. +4 −5 dojo/tools/generic/parser.py
  42. +1 −0 dojo/tools/mobsf/__init__.py
  43. +144 −0 dojo/tools/mobsf/parser.py
  44. +37 −16 dojo/tools/ssllabs/parser.py
  45. +8 −2 dojo/urls.py
  46. +1 −0 dojo/utils.py
  47. +140 −83 entrypoint_scripts/common/dojo-shared-resources.sh
  48. +1 −1 {batch_mode → entrypoint_scripts/common}/setup-superuser.expect
  49. +24 −0 entrypoint_scripts/deploy/post-action.bash
  50. +7 −0 entrypoint_scripts/misc/url_db.py
  51. +83 −0 entrypoint_scripts/run/startup-docker.bash
  52. +21 −4 entrypoint_scripts/test/travis-integration-test.sh
  53. +7 −10 entrypoint_scripts/test/travis-smoke-test.sh
  54. +4 −0 entrypoint_scripts/test/travis-unit-test.sh
  55. +7 −3 requirements.txt
  56. +67 −0 setup-docker.bash
  57. +8 −27 setup.bash
  58. +10 −8 setup.py
  59. +11 −2 tests/check_status.py
  60. +19 −35 tests/smoke_test.py
@@ -0,0 +1,4 @@
.git
.gitignore
*.md
.env*
@@ -70,6 +70,7 @@ dojo/uploads/threat/*
*.sqlite
*.db
celerybeat.pid
*.env*

weekly.txt
Monthly.txt
@@ -91,3 +92,4 @@ dojo/media
.venv/
venv/
ENV/
quick.bash
@@ -2,30 +2,27 @@ sudo: required
language: python
install: true

services:
- docker

before_script:
- export -f travis_fold
- export REPO=appsecpipeline/django-defectdojo
- export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi`

env:
- TEST=smoke-test
- TEST=integration-test
- TEST=unit-test
- TEST=bandit
- TEST=docker-bench-security
- TEST=ansible
- TEST=flake8-complete
- TEST=flake8
- TEST=safety

matrix:
allow_failures:
- env: TEST=smoke-test
- env: TEST=bandit
- env: TEST=sourceclear

services:
- docker

before_script:
- export -f travis_fold
- export REPO=appsecpipeline/django-defectdojo
- export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi`
- env: TEST=flake8-complete

script:
- |
@@ -46,27 +43,20 @@ script:
bash entrypoint_scripts/test/travis-integration-test.sh || exit 1
travis_fold end "integration-test"
;;
sourceclear)
## Run the SRC:CLR Scan
curl -sSL https://download.sourceclear.com/ci.sh | bash
;;
bandit)
# install bandit
pip install bandit
## Run Bandit python static code
bandit -r * -x venv,tests,ansible
bandit -r * -x venv,tests
;;
docker-bench-security)
## Run Docker Bench for Security
git clone https://github.com/docker/docker-bench-security.git
cd docker-bench-security
sh docker-bench-security.sh
;;
ansible)
true
;;
pep8)
flake8-complete)
pip install flake8
flake8 .
;;
@@ -90,12 +80,7 @@ script:
after_success:
#Push to docker repo
- |
if [ "$TRAVIS_TAG" != "" ] && [ "$DOCKER_USER" != "" ] && [ "$DOCKER_PASS" != "" ]; then
docker tag $REPO $REPO:$TRAVIS_TAG
docker login -u "$DOCKER_USER" -p "$DOCKER_PASS";
docker push $REPO ;
fi
- bash entrypoint_scripts/deploy/post-action.bash

notifications:
slack:
@@ -1,53 +1,28 @@
FROM ubuntu:16.04
MAINTAINER Matt Tesauro <matt.tesauro@owasp.org>
FROM ubuntu:16.04 as base
MAINTAINER Matt Tesauro <matt.tesauro@owasp.org>, Aaron Weaver <aaron.weaver@owasp.org>

# # # Create a single Docker image running DefectDojo and all dependencies
# # # Create a docker image for DefectDojo and all dependencies

# Setup database environment variables. Used to setup an external
# database, and is optional.
# Set a variable using build args.
# i.e. `docker build --build-arg DBNAME="db.foopy.com" ...`

ARG SQLHOST=""
ARG SQLPORT=""
ARG SQLUSER=""
ARG SQLPWD=""
ARG DBNAME=""

ENV SQLHOST=$SQLHOST
ENV SQLPORT=$SQLPORT
ENV SQLUSER=$SQLUSER
ENV SQLPWD=$SQLPWD
ENV DBNAME=$DBNAME

# Update and install basic requirements;
# Install mysql-server already at this place, since we want to avoid
# interactivity when creating a Docker image;
# Also: create the application user;
RUN apt-get update \
&& apt-get install -y sudo git expect wget \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server \
&& adduser --disabled-password --gecos "DefectDojo" dojo

# Give the app user sudo permissions and switch executing user
ADD ./docker/etc/dojo_sudo /etc/sudoers.d/
USER dojo:dojo
# Create the application user;
RUN adduser --disabled-password --gecos "DefectDojo" dojo

# Add the application files and start the setup
ADD --chown=dojo:dojo . /opt/django-DefectDojo
WORKDIR /opt/django-DefectDojo
# Add the -y option to avoid interactive prompts
RUN ./setup.bash -y

# Install wkhtmltopdf
RUN wget -O /tmp/wkhtmltox.tar.xz https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
&& tar xvfJ /tmp/wkhtmltox.tar.xz -C /tmp \
&& sudo chown root:root /tmp/wkhtmltox/bin/wkhtmltopdf \
&& sudo cp /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf
# Update and install basic requirements
RUN ./setup-docker.bash -y dependencies

# Start the DB server and rund the app
ENTRYPOINT sudo chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \
&& sudo service mysql start \
&& (celery -A dojo worker -l info --concurrency 3 >> /opt/django-DefectDojo/worker.log 2>&1 &) \
&& (celery beat -A dojo -l info >> /opt/django-DefectDojo/beat.log 2>&1 &) \
&& (python manage.py runserver 0.0.0.0:8000 >> /opt/django-DefectDojo/dojo.log 2>&1)
########## Stage: dev-mysql-self-contained ##########
FROM base as dev-mysql-self-contained
RUN ./setup-docker.bash -y db -d MYSQL
# Give the app user sudo permissions and switch executing user
ADD ./docker/etc/dojo_sudo /etc/sudoers.d/
# Start DefectDojo Services
CMD entrypoint_scripts/run/startup-docker.bash

########## Stage: release ##########
FROM dev-mysql-self-contained as release
RUN ./setup-docker.bash -y release
# USER dojo
CMD gunicorn --bind 0.0.0.0:$PORT wsgi
@@ -2,5 +2,6 @@ Please submit your pull requests to the 'dev' branch.

When submitting a pull request, please make sure you have completed the following checklist:

- [ ] Your code is flake8 compliant (Dojo's code isn't currently flake8 compliant, but we're trying to correct that)
- [ ] If this is a new feature and not a bug fix, you've included the proper documentation under the /docs folder
- [ ] Your code is flake8 compliant (DefectDojo's code isn't currently flake8 compliant, but we're trying to correct that.)
- [ ] If this is a new feature and not a bug fix, you've included the proper documentation in the ReadTheDocs documentation folder. https://github.com/DefectDojo/Documentation/tree/master/docs or provide feature documentation in the PR.
- [ ] Add applicable tests to the unit tests.
@@ -44,13 +44,13 @@ that should give you an idea of how to use DefectDojo for your own team.

- DefectDojo Python API: `pip install defectdojo_api` or clone the [repository](https://github.com/aaronweaver/defectdojo_api).

- Browse the API on [SwaggerHub](https://app.swaggerhub.com/apis/DefectDojo/defect-dojo_api_v_2/1.0.0). [![Swagger Status](http://online.swagger.io/validator?url=https://api.swaggerhub.com/apis/DefectDojo/defect-dojo_api_v_2/1.0.0)](https://app.swaggerhub.com/apis/DefectDojo/defect-dojo_api_v_2/1.0.0)
- Browse the API on [SwaggerHub](https://app.swaggerhub.com/apis/DefectDojo/defect-dojo_api_v_2/1.0.0). [![Swagger Status](http://online.swagger.io/validator?url=https://api.swaggerhub.com/apis/DefectDojo/defect-dojo_api_v_2/1.0.0)](https://app.swaggerhub.com/apis/DefectDojo/defect-dojo_api_v_2/1.0.0)

# Getting Involved

![Slack](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/slack_rgb.png)

Realtime discussion is done in the OWASP Slack Channel, #defectdojo. [Get Access.](https://owasp.herokuapp.com/)
Realtime discussion is done in the OWASP Slack Channel, #defectdojo. [Get Access.](http://owaspslack.com/)

![Twitter](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/Twitter_Logo.png)

This file was deleted.

@@ -18,7 +18,7 @@
"@yarn_components/fullcalendar": "fullcalendar/fullcalendar#*",
"@yarn_components/google-code-prettify": "tcollard/google-code-prettify#~1.0.4",
"@yarn_components/holderjs": "imsky/holder#~2.4.1",
"@yarn_components/jquery": "jquery/jquery-dist#~2.1.4",
"@yarn_components/jquery": "jquery/jquery-dist#~3.3.1",
"@yarn_components/jquery-cookie": "carhartl/jquery-cookie#*",
"@yarn_components/jquery-highlight": "knownasilya/jquery-highlight#*",
"@yarn_components/jquery-ui": "components/jqueryui#*",
@@ -4,7 +4,7 @@
# Django starts so that shared_task will use this app.
from .celery import app as celery_app # noqa

__version__ = '1.5.3'
__version__ = '1.5.4'
__url__ = 'https://github.com/DefectDojo/django-DefectDojo'
__docs__ = 'http://defectdojo.readthedocs.io/'
__demo__ = 'http://defectdojo.pythonanywhere.com/'
@@ -0,0 +1,4 @@
from auditlog.models import LogEntry
from django.contrib import admin

admin.site.unregister(LogEntry)
@@ -1,9 +1,9 @@
from dojo.models import Product, Engagement_Type, Engagement, Test, Finding, \
from dojo.models import Product, Engagement, Test, Finding, \
User, ScanSettings, IPScan, Scan, Stub_Finding, Risk_Acceptance, \
Finding_Template, Test_Type, Development_Environment, Report_Type, \
Finding_Template, Test_Type, Development_Environment, \
JIRA_Issue, Tool_Product_Settings, Tool_Configuration, Tool_Type, \
Product_Type, JIRA_Conf, Endpoint, BurpRawRequestResponse, JIRA_PKey, \
Notes, Dojo_User, Regulation, DojoMeta
Notes, DojoMeta
from dojo.forms import ImportScanForm, SEVERITY_CHOICES
from dojo.tools.factory import import_parser_factory
from django.core.validators import URLValidator, validate_ipv46_address
@@ -17,7 +17,6 @@
import json



class TagList(list):
def __init__(self, *args, **kwargs):
pretty_print = kwargs.pop("pretty_print", True)
@@ -117,6 +117,7 @@ class DojoMetaViewSet(mixins.ListModelMixin,
serializer_class = serializers.MetaSerializer
queryset = DojoMeta.objects.all()


class ProductViewSet(mixins.ListModelMixin,
mixins.RetrieveModelMixin,
mixins.CreateModelMixin,
@@ -131,7 +132,6 @@ class ProductViewSet(mixins.ListModelMixin,
# TODO: findings count field
filter_fields = ('id', 'name', 'prod_type', 'created', 'authorized_users')


def get_queryset(self):
if not self.request.user.is_staff:
return Product.objects.filter(

0 comments on commit 4f66fb7

Please sign in to comment.
You can’t perform that action at this time.