Skip to content
Permalink
Browse files

Stable python3 build

  • Loading branch information...
Maffooch Maffooch
Maffooch authored and Maffooch committed Jun 6, 2019
1 parent 8c558b8 commit 6508e7ec3883940df692fbe428d0d095ff442146
Showing with 815 additions and 646 deletions.
  1. +6 −7 Dockerfile.django
  2. +5 −5 Dockerfile.nginx
  3. +1 −0 docker-compose.override.yml
  4. +1 −1 docker/entrypoint-celery-beat.sh
  5. +1 −1 docker/entrypoint-celery-worker.sh
  6. +15 −14 docker/entrypoint-initializer.sh
  7. +3 −3 docker/entrypoint-unit-tests-devDocker.sh
  8. +17 −17 docker/entrypoint.sh
  9. +1 −1 dojo/__init__.py
  10. +8 −8 dojo/api.py
  11. +4 −4 dojo/api_v2/serializers.py
  12. +1 −1 dojo/benchmark/urls.py
  13. +2 −2 dojo/celery.py
  14. +1 −1 dojo/cred/urls.py
  15. +3 −3 dojo/cred/views.py
  16. +171 −172 dojo/db_migrations/0001_initial.py
  17. +1 −1 dojo/db_migrations/0002_auto_20190503_1817.py
  18. +1 −1 dojo/db_migrations/0003_test_title.py
  19. +1 −1 dojo/db_migrations/0004_cve_field.py
  20. +1 −1 dojo/db_migrations/0005_repo_field.py
  21. +1 −1 dojo/endpoint/views.py
  22. +3 −2 dojo/engagement/views.py
  23. +17 −22 dojo/filters.py
  24. +1 −1 dojo/finding/views.py
  25. +6 −6 dojo/forms.py
  26. +1 −1 dojo/jira_link/urls.py
  27. +1 −1 dojo/jira_link/views.py
  28. +1 −1 dojo/management/commands/csv_findings_export.py
  29. +2 −2 dojo/management/commands/dedupe.py
  30. +5 −5 dojo/management/commands/dupecheck.py
  31. +5 −5 dojo/management/commands/jira_async_updates.py
  32. +1 −1 dojo/management/commands/migrate_finding_templates.py
  33. +2 −2 dojo/management/commands/migrate_product_contacts.py
  34. +3 −3 dojo/management/commands/notify_isoc.py
  35. +2 −2 dojo/management/commands/push_to_jira_update.py
  36. +16 −20 dojo/management/commands/run_scan.py
  37. +4 −3 dojo/metrics/views.py
  38. +195 −17 dojo/models.py
  39. +1 −1 dojo/notes/urls.py
  40. +1 −1 dojo/notifications/urls.py
  41. +1 −1 dojo/object/parser.py
  42. +1 −1 dojo/object/urls.py
  43. +1 −1 dojo/product/views.py
  44. +5 −5 dojo/reports/views.py
  45. +27 −27 dojo/reports/widgets.py
  46. +7 −7 dojo/rules/views.py
  47. +1 −1 dojo/system_settings/urls.py
  48. +3 −3 dojo/tasks.py
  49. +5 −5 dojo/templatetags/display_tags.py
  50. +1 −1 dojo/templatetags/event_tags.py
  51. +1 −0 dojo/test/views.py
  52. +1 −1 dojo/tool_config/urls.py
  53. +1 −1 dojo/tool_product/urls.py
  54. +1 −1 dojo/tool_type/urls.py
  55. +3 −3 dojo/tools/acunetix/parser.py
  56. +2 −2 dojo/tools/acunetix/parser_helper.py
  57. +2 −2 dojo/tools/anchore_engine/parser.py
  58. +4 −4 dojo/tools/appspider/parser.py
  59. +6 −6 dojo/tools/arachni/parser.py
  60. +3 −3 dojo/tools/awsprowler/parser.py
  61. +6 −6 dojo/tools/awsscout2/parser.py
  62. +1 −1 dojo/tools/bandit/parser.py
  63. +1 −1 dojo/tools/blackduck/parser.py
  64. +1 −1 dojo/tools/brakeman/parser.py
  65. +1 −1 dojo/tools/bundler_audit/parser.py
  66. +5 −5 dojo/tools/burp/parser.py
  67. +1 −1 dojo/tools/checkmarx/parser.py
  68. +1 −1 dojo/tools/clair/parser.py
  69. +1 −1 dojo/tools/clair_klar/parser.py
  70. +3 −3 dojo/tools/cobalt/parser.py
  71. +1 −1 dojo/tools/contrast/parser.py
  72. +1 −1 dojo/tools/dawnscanner/parser.py
  73. +5 −5 dojo/tools/dependencycheck/parser.py
  74. +1 −1 dojo/tools/fortify/parser.py
  75. +4 −4 dojo/tools/generic/parser.py
  76. +1 −1 dojo/tools/gosec/parser.py
  77. +2 −2 dojo/tools/immuniweb/parser.py
  78. +4 −4 dojo/tools/kiuwan/parser.py
  79. +3 −3 dojo/tools/mobsf/parser.py
  80. +2 −2 dojo/tools/nessus/parser.py
  81. +1 −1 dojo/tools/netsparker/parser.py
  82. +3 −3 dojo/tools/nexpose/parser.py
  83. +2 −2 dojo/tools/nikto/parser.py
  84. +1 −1 dojo/tools/nmap/parser.py
  85. +2 −2 dojo/tools/npmaudit/parser.py
  86. +1 −1 dojo/tools/nsp/parser.py
  87. +2 −2 dojo/tools/openscap/parser.py
  88. +4 −4 dojo/tools/openvas_csv/parser.py
  89. +2 −2 dojo/tools/php_security_audit_v2/parser.py
  90. +4 −4 dojo/tools/phpsymfonysecuritycheck/parser.py
  91. +10 −10 dojo/tools/qualys/parser.py
  92. +3 −3 dojo/tools/qualys/utfdictcsv.py
  93. +8 −8 dojo/tools/qualyswebapp/parser.py
  94. +3 −3 dojo/tools/qualyswebapp/utfdictcsv.py
  95. +1 −1 dojo/tools/retirejs/parser.py
  96. +4 −4 dojo/tools/safety/parser.py
  97. +3 −3 dojo/tools/skf/parser.py
  98. +1 −1 dojo/tools/snyk/parser.py
  99. +1 −1 dojo/tools/sonatype/parser.py
  100. +1 −1 dojo/tools/spotbugs/parser.py
  101. +1 −1 dojo/tools/ssllabs/parser.py
  102. +1 −1 dojo/tools/trufflehog/parser.py
  103. +7 −7 dojo/tools/trustwave_csv/parser.py
  104. +1 −1 dojo/tools/twistlock/parser.py
  105. +4 −4 dojo/tools/vcg/parser.py
  106. +1 −1 dojo/tools/veracode/parser.py
  107. +3 −3 dojo/tools/wapiti/parser.py
  108. +1 −1 dojo/tools/zap/parser.py
  109. +3 −3 dojo/unittests/test_api_v1.py
  110. +4 −4 dojo/unittests/test_rest_framework.py
  111. +2 −2 dojo/unittests/test_vcg_parser.py
  112. +3 −0 dojo/user/urls.py
  113. +6 −4 dojo/user/views.py
  114. +5 −5 dojo/utils.py
  115. +3 −3 entrypoint_scripts/misc/url_db.py
  116. +30 −30 requirements.txt
  117. +3 −3 setup/scripts/misc/url_db.py
  118. +1 −1 tests/Test_selenium_unittest.py
  119. +1 −1 tests/apply_finding_template_tests.py
  120. +1 −1 tests/check_status.py
  121. +26 −34 tests/check_status_ui.py
  122. BIN tests/chromedriver
  123. +1 −1 tests/product_selenium_unittest.py
  124. +1 −1 tests/validate_acunetix_scan_xml.py
  125. +16 −16 tests/zap.py
@@ -4,7 +4,7 @@
# The code for the build image should be idendical with the code in
# Dockerfile.nginx to use the caching mechanism of Docker.

FROM python:2 as build
FROM python:3 as build
WORKDIR /app
RUN \
apt-get -y update && \
@@ -18,9 +18,9 @@ RUN \
rm -rf /var/lib/apt/lists && \
true
COPY requirements.txt ./
RUN pip wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt

FROM python:2-slim
FROM python:3-slim
WORKDIR /app
RUN \
apt-get -y update && \
@@ -41,10 +41,10 @@ RUN \
apt-get clean && \
rm -rf /var/lib/apt/lists && \
true
RUN pip install --no-cache-dir --upgrade pip
RUN pip3 install --no-cache-dir --upgrade pip
COPY --from=build /tmp/wheels /tmp/wheels
COPY requirements.txt ./
RUN pip install \
RUN pip3 install \
--no-cache-dir \
--no-index \
--find-links=/tmp/wheels \
@@ -56,7 +56,6 @@ COPY \
docker/entrypoint-uwsgi.sh \
docker/entrypoint-uwsgi-dev.sh \
docker/entrypoint-unit-tests.sh \
docker/entrypoint-unit-tests-devDocker.sh \
docker/wait-for-it.sh \
/
COPY wsgi.py manage.py tests/unit-tests.sh ./
@@ -70,7 +69,7 @@ RUN \
chmod g=u dojo/migrations && \
chmod g=u /var/run && \
true
USER 1001
USER root
ENV \
DD_ADMIN_USER=admin \
DD_ADMIN_MAIL=admin@defectdojo.local \
@@ -3,7 +3,7 @@
# The code for the build image should be idendical with the code in
# Dockerfile.django to use the caching mechanism of Docker.

FROM python:2 as build
FROM python:3 as build
WORKDIR /app
RUN \
apt-get -y update && \
@@ -17,7 +17,7 @@ RUN \
rm -rf /var/lib/apt/lists && \
true
COPY requirements.txt ./
RUN pip wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt

FROM build AS collectstatic

@@ -36,7 +36,7 @@ RUN \
rm -rf /var/lib/apt/lists && \
true

RUN pip install \
RUN pip3 install \
--no-cache-dir \
--no-index \
--find-links=/tmp/wheels \
@@ -51,7 +51,7 @@ RUN \
cd components && \
yarn && \
cd .. && \
python manage.py collectstatic && \
python3 manage.py collectstatic && \
true

FROM nginx
@@ -66,5 +66,5 @@ ENV \
DD_UWSGI_PASS="uwsgi_server" \
DD_UWSGI_HOST="uwsgi" \
DD_UWSGI_PORT="3031"
USER 1001
USER root
ENTRYPOINT ["/entrypoint-nginx.sh"]
@@ -3,7 +3,7 @@
umask 0002

echo -n "Waiting for database to be reachable "
until echo "select 1;" | python manage.py dbshell > /dev/null
until echo "select 1;" | python3 manage.py dbshell > /dev/null
do
echo -n "."
sleep 1
@@ -3,7 +3,7 @@
umask 0002

echo -n "Waiting for database to be reachable "
until echo "select 1;" | python manage.py dbshell > /dev/null
until echo "select 1;" | python3 manage.py dbshell > /dev/null
do
echo -n "."
sleep 1
@@ -19,14 +19,15 @@ then
fi

echo -n "Waiting for database to be reachable "
until echo "select 1;" | python manage.py dbshell > /dev/null
until echo "select 1;" | python3 manage.py dbshell > /dev/null
do
echo -n "."
sleep 1
done
echo

python manage.py migrate
python3 manage.py makemigrations dojo
python3 manage.py migrate

ADMIN_EXISTS=$(echo "SELECT * from auth_user;" | python manage.py dbshell | grep admin)

@@ -44,16 +45,16 @@ User.objects.create_superuser(
)
EOD

python manage.py loaddata product_type
python manage.py loaddata test_type
python manage.py loaddata development_environment
python manage.py loaddata system_settings
python manage.py loaddata benchmark_type
python manage.py loaddata benchmark_category
python manage.py loaddata benchmark_requirement
python manage.py loaddata language_type
python manage.py loaddata objects_review
python manage.py loaddata regulation
python manage.py installwatson
exec python manage.py buildwatson
python3 manage.py loaddata product_type
python3 manage.py loaddata test_type
python3 manage.py loaddata development_environment
python3 manage.py loaddata system_settings
python3 manage.py loaddata benchmark_type
python3 manage.py loaddata benchmark_category
python3 manage.py loaddata benchmark_requirement
python3 manage.py loaddata language_type
python3 manage.py loaddata objects_review
python3 manage.py loaddata regulation
python3 manage.py installwatson
exec python3 manage.py buildwatson
fi
@@ -5,10 +5,10 @@ cd /app
# Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)
unset DD_DATABASE_URL

python manage.py makemigrations dojo
python manage.py migrate
python3 manage.py makemigrations dojo
python3 manage.py migrate

python manage.py test dojo.unittests --keepdb
python3 manage.py test dojo.unittests --keepdb

echo "End of tests. Leaving the container up"
tail -f /dev/null
@@ -12,35 +12,35 @@ if [ -z "$DD_DATABASE_URL" ]; then
fi

if [ ! -f "/opt/django-DefectDojo/static/docker_complete" ]; then
python manage.py makemigrations dojo
python manage.py makemigrations --merge --noinput
python manage.py migrate
python3 manage.py makemigrations dojo
python3 manage.py makemigrations --merge --noinput
python3 manage.py migrate

if [ -z "$DD_ADMIN_PASSWORD" ]; then
DD_ADMIN_PASSWORD="admin"
fi

# The '&&' is critical here. If the admin user is already created, setting the
# password will not be done.
python manage.py createsuperuser \
python3 manage.py createsuperuser \
--noinput \
--username=admin \
--email='admin@localhost' && \
./docker/setup-superuser.expect

python manage.py loaddata product_type
python manage.py loaddata test_type
python manage.py loaddata development_environment
python manage.py loaddata system_settings
python manage.py loaddata benchmark_type
python manage.py loaddata benchmark_category
python manage.py loaddata benchmark_requirement
python manage.py loaddata language_type
python manage.py loaddata objects_review
python manage.py loaddata regulation
python manage.py installwatson
python manage.py buildwatson
python manage.py collectstatic --noinput
python3 manage.py loaddata product_type
python3 manage.py loaddata test_type
python3 manage.py loaddata development_environment
python3 manage.py loaddata system_settings
python3 manage.py loaddata benchmark_type
python3 manage.py loaddata benchmark_category
python3 manage.py loaddata benchmark_requirement
python3 manage.py loaddata language_type
python3 manage.py loaddata objects_review
python3 manage.py loaddata regulation
python3 manage.py installwatson
python3 manage.py buildwatson
python3 manage.py collectstatic --noinput
touch /opt/django-DefectDojo/static/docker_complete
fi

@@ -1,4 +1,4 @@
from __future__ import absolute_import


# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
@@ -30,7 +30,7 @@
Development_EnvironmentForm, Product_TypeForm, Test_TypeForm
from dojo.tools.factory import import_parser_factory
from datetime import datetime
from object.parser import import_object_eng
from .object.parser import import_object_eng

"""
Setup logging for the api
@@ -73,7 +73,7 @@ def form_args(self, bundle):
rsc = self.resource()
kwargs = super(ModelFormValidation, self).form_args(bundle)

for name, rel_field in rsc.fields.items():
for name, rel_field in list(rsc.fields.items()):
data = kwargs['data']
if not issubclass(rel_field.__class__, RelatedField):
continue # Not a resource field
@@ -1314,7 +1314,7 @@ def is_valid(self, bundle, request=None):
get_pk_from_uri(uri=bundle.data['engagement'])
except NotFound:
errors.setdefault('engagement', []).append('A valid engagement must be supplied. Ex. /api/v1/engagements/1/')
scan_type_list = list(map(lambda x: x[0], ImportScanForm.SCAN_TYPE_CHOICES))
scan_type_list = list([x[0] for x in ImportScanForm.SCAN_TYPE_CHOICES])
if 'scan_type' in bundle.data:
if bundle.data['scan_type'] not in scan_type_list:
errors.setdefault('scan_type', []).append('scan_type must be one of the following: ' + ', '.join(scan_type_list))
@@ -1329,7 +1329,7 @@ def is_valid(self, bundle, request=None):
errors.setdefault('test_type', []).append(
'test_type must be one of the following: ' +
', '.join(Test_Type.objects.values_list("name", flat=True)))
severity_list = list(map(lambda x: x[0], SEVERITY_CHOICES))
severity_list = list([x[0] for x in SEVERITY_CHOICES])
if 'minimum_severity' in bundle.data:
if bundle.data['minimum_severity'] not in severity_list:
errors.setdefault('minimum_severity', []).append('minimum_severity must be one of the following: ' + ', '.join(severity_list))
@@ -1457,8 +1457,8 @@ def obj_create(self, bundle, **kwargs):
try:
t.full_clean()
except ValidationError as e:
print "Error Validating Test Object"
print e
print("Error Validating Test Object")
print(e)

t.save()
t.tags = bundle.data['tags']
@@ -1556,13 +1556,13 @@ def is_valid(self, bundle, request=None):
get_pk_from_uri(uri=bundle.data['test'])
except NotFound:
errors.setdefault('test', []).append('A valid test must be supplied. Ex. /api/v1/tests/1/')
scan_type_list = list(map(lambda x: x[0], ImportScanForm.SCAN_TYPE_CHOICES))
scan_type_list = list([x[0] for x in ImportScanForm.SCAN_TYPE_CHOICES])
if 'scan_type' in bundle.data:
if bundle.data['scan_type'] not in scan_type_list:
errors.setdefault('scan_type', []).append('scan_type must be one of the following: ' + ', '.join(scan_type_list))
else:
errors.setdefault('scan_type', []).append('A scan_type must be given so we know how to import the scan file.')
severity_list = list(map(lambda x: x[0], SEVERITY_CHOICES))
severity_list = list([x[0] for x in SEVERITY_CHOICES])
if 'minimum_severity' in bundle.data:
if bundle.data['minimum_severity'] not in severity_list:
errors.setdefault('minimum_severity', []).append('minimum_severity must be one of the following: ' + ', '.join(severity_list))
@@ -115,7 +115,7 @@ def update(self, instance, validated_data):
return self._save_tags(tag_object, to_be_tagged)

def _save_tags(self, tag_object, tags):
for key in tags.keys():
for key in list(tags.keys()):
tag_values = tags.get(key)
tag_object.tags = ", ".join(tag_values)

@@ -124,7 +124,7 @@ def _save_tags(self, tag_object, tags):
def _pop_tags(self, validated_data):
to_be_tagged = {}

for key in self.fields.keys():
for key in list(self.fields.keys()):
field = self.fields[key]
if isinstance(field, TagListSerializerField):
if key in validated_data:
@@ -257,7 +257,7 @@ def validate(self, data):
host = data.get('host', self.instance.host)
product = data.get('product', None)

from urlparse import urlunsplit
from urllib.parse import urlunsplit
if protocol:
endpoint = urlunsplit((protocol, host, path, query, fragment))
else:
@@ -546,7 +546,7 @@ def save(self):
pass

test.save()
test.tags = u' '.join(data['tags'])
test.tags = ' '.join(data['tags'])
try:
parser = import_parser_factory(data['file'],
test,
@@ -1,7 +1,7 @@
from django.conf.urls import url
from django.contrib import admin
from django.apps import apps
import views
from . import views

urlpatterns = [
url(r'^benchmark/(?P<pid>\d+)/type/(?P<type>\d+)$', views.benchmark_view, name='view_product_benchmark'),
@@ -1,4 +1,4 @@
from __future__ import absolute_import

import os
from celery import Celery
from django.conf import settings
@@ -17,4 +17,4 @@

@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
print(('Request: {0!r}'.format(self.request)))
@@ -1,7 +1,7 @@
from django.conf.urls import url
from django.contrib import admin
from django.apps import apps
import views
from . import views

urlpatterns = [
url(r'^cred/add', views.new_cred, name='add_cred'),
@@ -557,10 +557,10 @@ def new_cred_finding(request, fid):
cred_id=cred_user.cred_id, finding=finding.id)

if cred_lookup:
print "Cred lookup valid"
print("Cred lookup valid")

if cred_user:
print "Cred user"
print("Cred user")

message = "Credential already associated."
status_tag = 'alert-danger'
@@ -703,7 +703,7 @@ def view_selenium(request, ttid):

mimetypes.init()
cred = Cred_Mapping.objects.get(pk=ttid)
print cred.cred_id.selenium_script
print(cred.cred_id.selenium_script)
# mimetype, encoding = mimetypes.guess_type(cred.cred_id.selenium_script)
response = StreamingHttpResponse(
FileIterWrapper(open(cred.cred_id.selenium_script)))

0 comments on commit 6508e7e

Please sign in to comment.
You can’t perform that action at this time.