From 5df8e241822fa80d1066cb135904d69438eb2460 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:53:22 -0800 Subject: [PATCH 01/51] Update pipeline-config.yaml (#440) --- pipeline-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 3c3e8bea..9d98c47f 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -9,54 +9,54 @@ stages: branches: - release unitTesting: - enabled: false + enabled: true branches: [] secretScanning: - enabled: false + enabled: true branches: - release sca: - enabled: false + enabled: true branches: - release codeLanguages: - Python - Javascript sast: - enabled: false + enabled: true branches: - release codeLanguages: - Python iac: - enabled: false + enabled: true branches: - release buildDocker: - enabled: false + enabled: true branches: - release containerScan: - enabled: false + enabled: true branches: - release containerName: secusphere containerTag: latest releaseToTest: - enabled: false + enabled: true branches: - release serviceName: secusphere containerTag: latest testRelease: - enabled: false + enabled: true branches: - release targetUrl: 'http://192.168.0.68:5010' dastTestType: full apiTargetUrl: 'http://192.168.0.68:5010/api/openapi.yaml' securityQualityGate: - enabled: false + enabled: true branches: - release deploy: @@ -83,7 +83,7 @@ stages: app.smtp.passwordRef: "SENDGRID-SMTP-PW" app.az.keyVaultName: "BkDevSecOpsKeyVault" post: - enabled: false + enabled: true branches: - release recipientEmails: 'brian@jbfinegoods.com' From c3fb035bf7c9b3c54717eb12a205942eef53905e Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 29 Dec 2023 21:09:41 -0800 Subject: [PATCH 02/51] Update tox.ini (#441) --- src/tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tox.ini b/src/tox.ini index 5c4f7a01..eedeee12 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -7,7 +7,7 @@ deps = pytest coverage pytest-cov - Flask==2.2.2 + Flask==2.3.3 Flask-SQLAlchemy==3.0.3 Flask-Login==0.6.2 Flask-Moment==1.0.5 @@ -15,11 +15,11 @@ deps = Flask-Markdown==0.3 Flask-Bootstrap==3.3.7.1 pyotp==2.8.0 - PyJWT==2.6.0 + PyJWT==2.7.0 pycryptodome==3.17 PyQRCode==1.2.1 python-dateutil==2.8.2 - requests==2.28.2 + requests==2.31.0 azure-identity==1.12.0 azure-keyvault-secrets==4.6.0 azure-keyvault-certificates==4.6.0 From 57a681aae8c644eb08cd9166fa2fdd2709583866 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 30 Dec 2023 00:04:00 -0800 Subject: [PATCH 03/51] Feature/fix toxi (#443) * Update tox.ini * fix unit test failures --- src/vr/templates/vulns/all_vulnerabilities_filtered.html | 4 ---- src/vr/vulns/web/findings.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/vr/templates/vulns/all_vulnerabilities_filtered.html b/src/vr/templates/vulns/all_vulnerabilities_filtered.html index 22bfd116..f67b34cb 100644 --- a/src/vr/templates/vulns/all_vulnerabilities_filtered.html +++ b/src/vr/templates/vulns/all_vulnerabilities_filtered.html @@ -51,11 +51,7 @@

Filter - - {% include "vulns/csv_upload_modal.html" %} - - {% include "vulns/csv_upload_explanation_modal.html" %} diff --git a/src/vr/vulns/web/findings.py b/src/vr/vulns/web/findings.py index 41a34307..8c9938c8 100644 --- a/src/vr/vulns/web/findings.py +++ b/src/vr/vulns/web/findings.py @@ -768,7 +768,7 @@ def finding(appid, id): else: finding_accuracy = 'N/A' referrer = request.referrer - if 'all_app_vulns_filtered/' in referrer: + if referrer and 'all_app_vulns_filtered/' in referrer: nav_bar = 'Application' else: nav_bar = 'Component' From b3d741cb7da844ed83cc6d73e33f44acb550097a Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 30 Dec 2023 00:33:34 -0800 Subject: [PATCH 04/51] Feature/fix toxi (#445) * Update tox.ini * fix unit test failures * Update web_testing.py --- ci_cd/unit_tests/web_testing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci_cd/unit_tests/web_testing.py b/ci_cd/unit_tests/web_testing.py index 126b4d45..fbe6e2f5 100644 --- a/ci_cd/unit_tests/web_testing.py +++ b/ci_cd/unit_tests/web_testing.py @@ -1632,7 +1632,9 @@ def test_edit_application_post(self): 'regulations': 1, } response = self._post_test_handler(route, data_dict) - assert response.status_code == 200 + match = _three_o_two_handler(response.headers, f"/edit_application/{app.ID}") + assert response.status_code == 302 + assert match def test_contacts_get(self): app = BusinessApplications.query.filter_by(ApplicationName=TEST_APP_NAME).first() From f62e0d15427b42649066bc7da6ac2120b17b24f5 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 30 Dec 2023 19:10:15 -0800 Subject: [PATCH 05/51] Feature/fix toxi (#447) * Update tox.ini * fix unit test failures * Update web_testing.py * Update Jenkinsfile --- Jenkinsfile | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 981681a1..b4e59984 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -299,21 +299,7 @@ pipeline { post { always { script { - def reportProcessor = new PipelineReportProcessor(this) - reportProcessor.processReport('pipeline_stage_report.json') - - def reportFile = 'pipeline_stage_report.json' - archiveArtifacts artifacts: reportFile, allowEmptyArchive: true - - def stageConfig = jslReadYamlConfig('post') - def recipientEmails = stageConfig?.recipientEmails - def recipientTeamsChannels = stageConfig?.recipientTeamsChannels - - jslSendMicrosoftTeamsPipelineReportMessage(recipientTeamsChannels) - jslSendMicrosoftTeamsMessage(recipientTeamsChannels) - jslSendPipelineStageReportEmail(recipientEmails) - jslSendSecurityReportEmail(recipientEmails) - jslSendPipelineReport() + jslPipelineReporter() } } } From 5c40b881a418ac78b4df7b8a94b5d22d1c61aea5 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:06:57 -0800 Subject: [PATCH 06/51] Feature/fix toxi (#449) * Update tox.ini * fix unit test failures * Update web_testing.py * Update Jenkinsfile * add dockerImg map for Container scans --- flask_endpoints.csv | 262 +++++++++++------------ src/vr/api/vulns/vulnerabilities.py | 24 ++- src/vr/orchestration/web/dockerimages.py | 2 + 3 files changed, 150 insertions(+), 138 deletions(-) diff --git a/flask_endpoints.csv b/flask_endpoints.csv index 7ce7e5bc..8d5bb91e 100644 --- a/flask_endpoints.csv +++ b/flask_endpoints.csv @@ -1,156 +1,156 @@ Endpoint,Methods -/create_client,GET -/api/documentation,GET -/forgotpw,"GET, POST" -/resetpw//,"GET, POST" -/forgotun,"GET, POST" -/displayun//,"GET, POST" /,GET -/login,"GET, POST" -,GET -/check_if_mfa,POST -/logout,GET -/logout,GET -/messages,"GET, POST" -/suppress_msg,POST -/onboarding,GET -/onboarding_suppress,GET -/register,GET -/register_user/,GET -/register_user_submit,POST -/register_submit,POST -/qrcode,GET -/settings,GET -/unauth_403,GET -/users,"GET, POST" -/add_user_role,POST -/remove_user_role,POST -/remove_user_appview_role,POST -/remove_user,POST +/add_app_integration/,GET +/add_application,"GET, POST" +/add_application_environment/,"GET, POST" +/add_benchmark_attachment,POST +/add_benchmark_note,POST +/add_cicd_pipeline/,"GET, POST" +/add_cicd_pipeline_stage/,GET +/add_contact/,POST +/add_integration,"GET, POST" +/add_issue_dispo,POST +/add_issue_note,POST /add_new_user,POST -/edit_profile,"GET, POST" -/update_mfa_status,POST -/display_mfa_qr,GET -/mobile_sync,GET -/profile,GET -/mobile_qrcode,GET -/mfa_qrcode,GET -/api/oauth/authorize,"GET, POST" -/api/oauth/token,POST -/api/oauth/revoke,POST -/api/openapi.yaml,GET -/api/onboard_new_application,POST +/add_service_ticket/,POST +/add_user_role,POST +/all_app_integrations/,GET +/all_app_vulns_filtered///,"GET, POST" +/all_app_vulns_filtered////csv,"GET, POST" +/all_app_vulns_filtered////export,"GET, POST" +/all_application_benchmarks/,"GET, POST" +/all_application_environments/,GET +/all_application_metrics,GET +/all_applications,"GET, POST" +/all_applications_filtered//,"GET, POST" +/all_cheatsheets,GET +/all_cicd_pipelines/,GET +/all_dockerimages,"GET, POST" +/all_git_repos,GET +/all_integrations,GET +/all_pipeline_jobs,"GET, POST" +/all_pipelines,GET +/all_regulations,GET +/all_service_tickets,"GET, POST" +/all_vulnerabilities,"GET, POST" +/all_vulnerabilities/csv,"GET, POST" +/all_vulnerabilities/export,"GET, POST" +/all_vulnerabilities_filtered//,"GET, POST" +/all_vulnerabilities_filtered///csv,"GET, POST" +/all_vulnerabilities_filtered///export,"GET, POST" +/analyze,POST +/api/add_application_profile/,POST /api/add_loc,POST +/api/add_sg_results,POST +/api/add_vulnerabilities,POST +/api/check_security_scan_status/,GET +/api/closeout_security_scan/,POST +/api/delete_vulnerabilities,POST +/api/documentation,GET +/api/edit_vulnerabilities,POST /api/get_application_profile/,GET -/api/add_application_profile/,POST -/api/jenkins_webhook,POST /api/jenkins_pipeline_reporter,POST +/api/jenkins_webhook,POST +/api/oauth/authorize,"GET, POST" +/api/oauth/revoke,POST +/api/oauth/token,POST +/api/onboard_new_application,POST +/api/openapi.yaml,GET /api/parallel_security_scan,POST -/api/check_security_scan_status/,GET -/api/closeout_security_scan/,POST -/api/add_sg_results,POST -/api/vulnerabilities,GET /api/search_vulnerabilities,POST -/api/add_vulnerabilities,POST -/api/edit_vulnerabilities,POST -/api/delete_vulnerabilities,POST -/all_application_benchmarks/,"GET, POST" -/benchmark_assessments/,"GET, POST" -/application_benchmarks//,"GET, POST" -/assessment_results//,GET -/add_benchmark_note,POST -/delete_benchmark_note,POST -/add_benchmark_attachment,POST -/delete_benchmark_attachment,POST -/download_benchmark_attachment/,GET -/submit_risk_profile,POST -/all_applications,"GET, POST" -/all_applications_filtered//,"GET, POST" +/api/vulnerabilities,GET +/applevel_metrics/,GET /application///,GET -/application//export,GET /application//csv,GET -/add_application,"GET, POST" -/application_issues/,"GET, POST" -/delete_application/,GET +/application//export,GET +/application_benchmarks//,"GET, POST" /application_endpoints/,"GET, POST" -/add_integration,"GET, POST" -/all_integrations,GET -/validate_integration,POST -/add_app_integration/,GET -/submit_app_integration/,POST -/all_app_integrations/,GET -/remove_app_integration,POST -/all_regulations,GET -/edit_application/,"GET, POST" -/add_application_environment/,"GET, POST" -/all_application_environments/,GET -/remove_application_environment,POST -/edit_application_environment//,"GET, POST" -/add_cicd_pipeline/,"GET, POST" -/all_cicd_pipelines/,GET -/remove_cicd_pipeline,POST -/contacts/,GET -/add_contact/,POST -/delete_contact,POST -/all_pipelines,GET -/pipeline_generator/,GET -/analyze,POST -/submit,POST -/all_dockerimages,"GET, POST" -/dockerimages/,"GET, POST" -/all_pipeline_jobs,"GET, POST" -/pipeline_jobs/,"GET, POST" -/add_cicd_pipeline_stage/,GET -/get_cicd_pipeline_stage_data,POST -/validate_cicd_pipeline_stage/,POST +/application_issues/,"GET, POST" +/application_KPIs/,"GET, POST" +/application_profile/,GET +/assessment_results//,GET +/benchmark_assessments/,"GET, POST" /branches/,"GET, POST" -/all_cheatsheets,GET /cheatsheets/,GET -/all_git_repos,GET +/check_if_mfa,POST +/component_KPIs/,"GET, POST" /components/,"GET, POST" -/all_service_tickets,"GET, POST" -/issue//,GET -/add_service_ticket/,POST -/sourcecode_files/,"GET, POST" -/threat_modeler/,"GET, POST" -/threat_assessments/,"GET, POST" -/threat_assessment//,GET +/contacts/,GET +/create_client,GET /dashboard,GET +/delete_application/,GET +/delete_benchmark_attachment,POST +/delete_benchmark_note,POST +/delete_contact,POST +/delete_issue_note,POST /devopsscorecard/,"GET, POST" +/display_mfa_qr,GET +/displayun//,"GET, POST" +/dockerimages/,"GET, POST" +/download_benchmark_attachment/,GET +/edit_application/,"GET, POST" +/edit_application_environment//,"GET, POST" +/edit_profile,"GET, POST" +/filtered_findings///,"GET, POST" +/filtered_findings////csv,"GET, POST" +/filtered_findings////export,"GET, POST" +/finding//,GET +/finding///request_review,GET +/forgotpw,"GET, POST" +/forgotun,"GET, POST" +/get_cicd_pipeline_stage_data,POST +/global_KPIs,"GET, POST" +/issue//,GET +/login,"GET, POST" +/logout,GET +/logout,GET +/messages,"GET, POST" +/metrics/,GET +/mfa_qrcode,GET +/mobile_qrcode,GET +/mobile_sync,GET +/on_demand_testing,POST +/onboarding,GET +/onboarding_suppress,GET /open_findings/,"GET, POST" -/open_findings//export,"GET, POST" /open_findings//csv,"GET, POST" +/open_findings//export,"GET, POST" /open_findings_for_scan//,"GET, POST" -/open_findings_for_scan///export,"GET, POST" /open_findings_for_scan///csv,"GET, POST" -/finding///request_review,GET -/finding//,GET -/filtered_findings///,"GET, POST" -/filtered_findings////export,"GET, POST" -/filtered_findings////csv,"GET, POST" -/add_issue_dispo,POST -/add_issue_note,POST -/delete_issue_note,POST -/metrics/,GET -/all_application_metrics,GET -/applevel_metrics/,GET -/application_KPIs/,"GET, POST" -/component_KPIs/,"GET, POST" -/global_KPIs,"GET, POST" +/open_findings_for_scan///export,"GET, POST" +/pipeline_generator/,GET +/pipeline_jobs/,"GET, POST" +/profile,GET +/qrcode,GET +/register,GET +/register_submit,POST +/register_user/,GET +/register_user_submit,POST +/remove_app_integration,POST +/remove_application_environment,POST +/remove_cicd_pipeline,POST +/remove_user,POST +/remove_user_appview_role,POST +/remove_user_role,POST +/resetpw//,"GET, POST" /securitygatescorecard/,GET /securitygatesettings,GET +/settings,GET +/sourcecode_files/,"GET, POST" +/submit,POST +/submit_app_integration/,POST +/submit_risk_profile,POST +/suppress_msg,POST +/threat_assessment//,GET +/threat_assessments/,"GET, POST" +/threat_modeler/,"GET, POST" +/unauth_403,GET +/update_mfa_status,POST /update_securitygatesettings,POST -/vulnerability_scans/,"GET, POST" -/on_demand_testing,POST -/application_profile/,GET +/users,"GET, POST" +/validate_cicd_pipeline_stage/,POST +/validate_integration,POST /visual_pipeline/,GET /visual_vulnerabilities/,GET -/all_vulnerabilities,"GET, POST" -/all_vulnerabilities/export,"GET, POST" -/all_vulnerabilities/csv,"GET, POST" -/all_vulnerabilities_filtered//,"GET, POST" -/all_vulnerabilities_filtered///export,"GET, POST" -/all_vulnerabilities_filtered///csv,"GET, POST" -/all_app_vulns_filtered///,"GET, POST" -/all_app_vulns_filtered////export,"GET, POST" -/all_app_vulns_filtered////csv,"GET, POST" +/vulnerability_scans/,"GET, POST" +,GET diff --git a/src/vr/api/vulns/vulnerabilities.py b/src/vr/api/vulns/vulnerabilities.py index abb71663..7f0c000e 100644 --- a/src/vr/api/vulns/vulnerabilities.py +++ b/src/vr/api/vulns/vulnerabilities.py @@ -22,6 +22,7 @@ ERROR_RESP = "Error: Invalid API Request" + @api.route("/api/vulnerabilities") @require_oauth('read:vulnerabilities') def get_vulnerabilities(): @@ -93,12 +94,21 @@ def update_vulnerabilities_status(app_cmdb_id, scan_id, req_raw): if i.ID not in scans_to_check: scans_to_check.append(i.ID) scans_to_check = sorted(scans_to_check, reverse=True) - - previous_vulns = Vulnerabilities\ - .query\ - .join(VulnerabilityScans, VulnerabilityScans.ID==Vulnerabilities.ScanId)\ - .filter(text(f"(Vulnerabilities.Status NOT LIKE 'Closed-%' OR Vulnerabilities.Status='Closed-Mitigated') AND (Vulnerabilities.ApplicationId='{app_cmdb_id}') AND (Vulnerabilities.SourceType='{scan_type.split('CI/CD-')[1]}') AND (Vulnerabilities.InitialScanId!='{scan_id}')"))\ - .all() + if req_raw['scanType'] == 'Container': + if 'dockerImg' in req_raw: + previous_vulns = Vulnerabilities \ + .query \ + .join(VulnerabilityScans, VulnerabilityScans.ID == Vulnerabilities.ScanId) \ + .join(DockerImages, DockerImages.ID == Vulnerabilities.DockerImageId) \ + .filter(text( + f"(Vulnerabilities.Status NOT LIKE 'Closed-%' OR Vulnerabilities.Status='Closed-Mitigated') AND (Vulnerabilities.ApplicationId='{app_cmdb_id}') AND (Vulnerabilities.SourceType='{scan_type.split('CI/CD-')[1]}') AND (Vulnerabilities.InitialScanId!='{scan_id}') AND (DockerImages.ImageName=='{req_raw['dockerImg']}')")) \ + .all() + else: + previous_vulns = Vulnerabilities\ + .query\ + .join(VulnerabilityScans, VulnerabilityScans.ID==Vulnerabilities.ScanId)\ + .filter(text(f"(Vulnerabilities.Status NOT LIKE 'Closed-%' OR Vulnerabilities.Status='Closed-Mitigated') AND (Vulnerabilities.ApplicationId='{app_cmdb_id}') AND (Vulnerabilities.SourceType='{scan_type.split('CI/CD-')[1]}') AND (Vulnerabilities.InitialScanId!='{scan_id}')"))\ + .all() closed_cnt = 0 new_vulns = req_raw['findings'] @@ -107,7 +117,7 @@ def update_vulnerabilities_status(app_cmdb_id, scan_id, req_raw): prev_id_check = i.VulnerabilityID for j in new_vulns: new_id_check = j['b_VulnerabilityID'] if 'b_VulnerabilityID' in j else None - if prev_id_check == new_id_check: + if (prev_id_check == new_id_check) and (i.SourceType == j['SourceType']): found = True break if not found and i.Status != "Closed-Mitigated": diff --git a/src/vr/orchestration/web/dockerimages.py b/src/vr/orchestration/web/dockerimages.py index c38f76df..4afb87ce 100644 --- a/src/vr/orchestration/web/dockerimages.py +++ b/src/vr/orchestration/web/dockerimages.py @@ -18,6 +18,7 @@ LOGIN_URL = "admin.login" UNAUTH_URL = "403.html" SERVER_ERR_URL = "500.html" +VULN_STATUS_IS_NOT_CLOSED = "Vulnerabilities.Status NOT LIKE 'Closed-%' AND Vulnerabilities.Status NOT LIKE 'Open-RiskAccepted-%'" @orchestration.route("/all_dockerimages", methods=['GET', 'POST']) @@ -128,6 +129,7 @@ def dockerimages(appid): DockerImages.ID, DockerImages.AddDate, DockerImages.ImageName, DockerImages.ImageTag, DockerImages.ImageId, DockerImages.AppIdList, func.count(Vulnerabilities.VulnerabilityID).label('total_vulnerabilities') ).join(Vulnerabilities, Vulnerabilities.DockerImageId == DockerImages.ID) \ + .filter(text(VULN_STATUS_IS_NOT_CLOSED)) \ .filter(text(f"DockerImages.AppIdList LIKE '%{appid},%'")) \ .group_by(DockerImages.ID) \ .order_by(text(orderby)) \ From b8d2775262ae6ac137916bb244a635fa18ef7bb0 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Wed, 3 Jan 2024 20:36:17 -0800 Subject: [PATCH 07/51] Update Jenkinsfile (#452) --- Jenkinsfile | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b4e59984..bb221109 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,8 +3,12 @@ pipeline { - agent any - + agent { + docker { + image 'securityuniversal/jenkins-pipeline-agent:latest' + args '--group-add 999' + } + } stages { stage('Initialize Config') { @@ -45,6 +49,11 @@ pipeline { } stage('Unit Testing') { + agent { + docker { + image 'securityuniversal/jenkins-python-agent:latest' + } + } when { expression { def config = jslReadYamlConfig('unitTesting') @@ -64,6 +73,11 @@ pipeline { } stage('Secret Scanning') { + agent { + docker { + image 'securityuniversal/jenkins-secret-agent:latest' + } + } when { expression { def config = jslReadYamlConfig('secretScanning') @@ -83,6 +97,11 @@ pipeline { } stage('Software Composition Analysis') { + agent { + docker { + image 'securityuniversal/jenkins-codetesting-agent:latest' + } + } when { expression { def config = jslReadYamlConfig('sca') @@ -106,6 +125,11 @@ pipeline { } stage('Static Application Security Testing') { + agent { + docker { + image 'securityuniversal/jenkins-codetesting-agent:latest' + } + } when { expression { def config = jslReadYamlConfig('sast') @@ -129,6 +153,12 @@ pipeline { } stage('Infrastructure-as-Code Security Testing') { + agent { + docker { + image 'securityuniversal/jenkins-iac-agent:latest' + args '--group-add 999' + } + } when { expression { def config = jslReadYamlConfig('iac') @@ -148,6 +178,12 @@ pipeline { } stage('Build Docker Service') { + agent { + docker { + image 'securityuniversal/jenkins-iac-agent:latest' + args '--group-add 999' + } + } when { expression { def config = jslReadYamlConfig('buildDocker') @@ -171,6 +207,12 @@ pipeline { } stage('Docker Container Scanning') { + agent { + docker { + image 'securityuniversal/jenkins-iac-agent:latest' + args '--group-add 999' + } + } when { expression { def config = jslReadYamlConfig('containerScan') @@ -195,6 +237,11 @@ pipeline { } stage('Release to Test') { + agent { + docker { + image 'securityuniversal/jenkins-deploy-agent:latest' + } + } when { expression { def config = jslReadYamlConfig('releaseToTest') @@ -266,6 +313,11 @@ pipeline { ////////// Deploy to Production ////////// stage('Deploy') { + agent { + docker { + image 'securityuniversal/jenkins-deploy-agent:latest' + } + } when { anyOf { // Condition for the PROD branch From 9e4030de31d47164737f216f7578d45fe04d7eb9 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 8 Jan 2024 07:53:01 -0800 Subject: [PATCH 08/51] Feature/jenkinsfile updates (#453) * Update Jenkinsfile * added scores and grades api endpoint --- src/vr/admin/routes/settings.py | 73 +++++- src/vr/api/__init__.py | 1 + src/vr/api/vulns/dora_extended.py | 378 +++++++++++++++++++++++++++ src/vr/db_models/setup.py | 6 +- src/vr/db_models/setup_2.py | 2 +- src/vr/templates/admin/settings.html | 97 ++++++- 6 files changed, 553 insertions(+), 4 deletions(-) create mode 100644 src/vr/api/vulns/dora_extended.py diff --git a/src/vr/admin/routes/settings.py b/src/vr/admin/routes/settings.py index ea761033..c366ce43 100644 --- a/src/vr/admin/routes/settings.py +++ b/src/vr/admin/routes/settings.py @@ -1,5 +1,7 @@ from flask import session, redirect, url_for, render_template from flask_login import login_required +from vr import db, app +import os # Start of Entity-specific Imports from vr.admin import admin from vr.admin.functions import _auth_user, check_menu_tour_init @@ -9,7 +11,7 @@ JENKINS_HOST, JENKINS_KEY, JENKINS_PROJECT, JENKINS_STAGING_PROJECT, JENKINS_TOKEN, SMTP_ADMIN_EMAIL, \ SMTP_HOST, SMTP_PASSWORD, SMTP_USER, SNOW_CLIENT_ID, SNOW_CLIENT_SECRET, SNOW_INSTANCE_NAME, SNOW_PASSWORD, \ SNOW_USERNAME, VERSION - +from flask_sqlalchemy import SQLAlchemy NAV = { 'CAT': { "name": "Settings", "url": "admin.admin_dashboard"} @@ -63,3 +65,72 @@ def settings(): } return render_template('admin/settings.html', user_roles=user_roles, NAV=NAV, user=user, settings=current_settings) + +@admin.route('/dangerous/delete_all', methods=['POST']) +def delete_all_data(): + NAV['curpage'] = {"name": "Settings"} + user, status, user_roles = _auth_user(session, 'No Role') + if status == 401: + return redirect(url_for('admin.login')) + elif status == 403: + return render_template('403.html', user=user, nav_cat={}, nav_subcat='', \ + nav_subsubcat='', nav_curpage={"name": "Unauthorized"}) + + try: + if ENV == 'test': + # Ensure all connections to the database are closed + db.session.close() + db.engine.dispose() + + # Path to the SQLite database file + db_path = "instance/" + app.config['SQLALCHEMY_DATABASE_URI'].replace('sqlite:///', '') + + # Delete the database file + if os.path.exists(db_path): + os.remove(db_path) + else: + print("The file does not exist") + else: + # Reflect the current database schema + db.reflect() + + # Retrieve all table names + table_names = db.engine.table_names() + + # Drop each table + for table_name in reversed(table_names): + db.engine.execute(f'DROP TABLE IF EXISTS {table_name} CASCADE') + + # Recreate all tables based on the current models + from vr.functions.mysql_db import connect_to_db + cur, db_obj = connect_to_db() + sql = '''CREATE TABLE "User" (id INTEGER NOT NULL, is_active BOOLEAN DEFAULT '1' NOT NULL, is_admin BOOLEAN DEFAULT '0' NOT NULL, is_security BOOLEAN DEFAULT '0' NOT NULL, username VARCHAR(100), password VARCHAR(255), auth_type VARCHAR(20), mfa_enabled BOOLEAN DEFAULT '0' NOT NULL, otp_secret VARCHAR(16), email VARCHAR(255) NOT NULL, email_confirmed_at DATETIME, first_name VARCHAR(100) DEFAULT '' NOT NULL, last_name VARCHAR(100) DEFAULT '' NOT NULL, jobtitle VARCHAR(100), dept VARCHAR(100), user_type VARCHAR(100), avatar_path VARCHAR(100), email_updates VARCHAR(1), app_updates VARCHAR(1), text_updates VARCHAR(1), registration_date DATETIME, loc_zipcode VARCHAR(20), loc_city VARCHAR(100), loc_state VARCHAR(50), about_me VARCHAR(2000), web_tz VARCHAR(100), phone_no VARCHAR(40), support_id VARCHAR(50), support_key VARCHAR(50), support_contact_id INTEGER, auth_token VARCHAR(300), onboarding_confirmed VARCHAR(1), PRIMARY KEY (id), UNIQUE (email))''' + cur.execute(sql) + db_obj.commit() + sql = 'CREATE TABLE "AppConfig" (id INTEGER NOT NULL, first_access BOOLEAN NOT NULL,PRIMARY KEY (id))' + cur.execute(sql) + db_obj.commit() + sql = 'CREATE TABLE "EntityPermissions" ("ID" INTEGER NOT NULL, "AddDate" DATETIME NOT NULL, "UserID" INTEGER, "EntityType" VARCHAR(100), "EntityID" VARCHAR(100), PRIMARY KEY ("ID"), FOREIGN KEY("UserID") REFERENCES "User" (id) ON DELETE CASCADE)' + cur.execute(sql) + db_obj.commit() + sql = 'CREATE TABLE "SourceCodeFile" ("ID" INTEGER NOT NULL, "AddDate" DATETIME, "GitRepoId" INTEGER, "FileName" VARCHAR(300), "FileLocation" VARCHAR(300), "FileType" VARCHAR(300), PRIMARY KEY ("ID"))' + cur.execute(sql) + db_obj.commit() + sql = 'CREATE TABLE "TmControls" ("ID" INTEGER NOT NULL, "AddDate" DATETIME NOT NULL, "Control" TEXT, "Type" VARCHAR(8), "Description" TEXT, "Lambda" VARCHAR(1), "Process" VARCHAR(1), "Server" VARCHAR(1), "Dataflow" VARCHAR(1), "Datastore" VARCHAR(1), "ExternalEntity" VARCHAR(1), PRIMARY KEY ("ID"))' + cur.execute(sql) + db_obj.commit() + sql = 'CREATE TABLE "UserRoleAssignments" (id INTEGER NOT NULL, user_id INTEGER, role_id INTEGER, PRIMARY KEY (id), FOREIGN KEY(user_id) REFERENCES "User" (id) ON DELETE CASCADE, FOREIGN KEY(role_id) REFERENCES "UserRoles" (id) ON DELETE CASCADE)' + cur.execute(sql) + db_obj.commit() + sql = 'CREATE TABLE "UserRoles" (id INTEGER NOT NULL, name VARCHAR(50), description VARCHAR(200), PRIMARY KEY (id), UNIQUE (name))' + cur.execute(sql) + db_obj.commit() + db_obj.close() + + + return "All tables dropped successfully", 200 + except Exception as e: + # Log the exception for debugging purposes + print(e) + db.session.rollback() + return "Error occurred during table deletion", 500 diff --git a/src/vr/api/__init__.py b/src/vr/api/__init__.py index 9c75fdb9..b6512480 100644 --- a/src/vr/api/__init__.py +++ b/src/vr/api/__init__.py @@ -10,5 +10,6 @@ from vr.api.vulns import vulnerabilities from vr.api.vulns import jenkins_webhook from vr.api.vulns import application_profiler +from vr.api.vulns import dora_extended from vr.api.integrations import servicenow diff --git a/src/vr/api/vulns/dora_extended.py b/src/vr/api/vulns/dora_extended.py new file mode 100644 index 00000000..1759a780 --- /dev/null +++ b/src/vr/api/vulns/dora_extended.py @@ -0,0 +1,378 @@ +import datetime +from vr import db, app +from flask import jsonify, request +from sqlalchemy import desc, text +from vr.api import api +from vr.admin.functions import db_connection_handler +from vr.admin.oauth2 import require_oauth +from authlib.integrations.flask_oauth2 import current_token +from vr.admin.auth_functions import verify_api_key, get_token_auth_header +from vr.functions.routing_functions import check_entity_permissions +from vr.assets.model.applicationprofiles import ApplicationProfiles, ApplicationProfilesSchema +from vr.assets.model.businessapplications import BusinessApplications +from vr.vulns.model.vulnerabilities import Vulnerabilities, MakeVulnerabilitiesSchema, VulnerabilitiesSchema +from vr.vulns.model.vulnerabilityscans import VulnerabilityScans +from vr.assessments.model.assessmentbenchmarkassessments import AssessmentBenchmarkAssessments +from vr.assessments.model.assessmentbenchmarkrules import AssessmentBenchmarkRules +from vr.assessments.model.assessmentbenchmarkruleaudits import AssessmentBenchmarkRuleAudits + + +ERROR_RESP = "Error: Invalid API Request" + +@api.route("/api/get_dora_grade/", methods=['POST', 'GET']) +@require_oauth('read:vulnerabilities') +def get_dora_grade(app_id): + token = current_token + auth, user_id, is_admin = verify_api_key(token) + response = jsonify({'response': ERROR_RESP}), 403 + if auth == 'valid': + permitted = check_entity_permissions(is_admin) + if permitted: + if request.method == 'POST': + form = request.get_json() + time_start = datetime.datetime.strptime(form['timeStart'], '%Y-%m-%d %H:%M:%S') + time_end = datetime.datetime.strptime(form['timeEnd'], '%Y-%m-%d %H:%M:%S') + else: + time_start = datetime.datetime.strptime('1990-01-01 12:00:00', '%Y-%m-%d %H:%M:%S') + time_end = datetime.datetime.utcnow() + time_dict = {'timeStart': time_start, 'timeEnd':time_end} + total_score = 0 + applicable_metrics = 0 + # calculate vulnerability resolution time + vr_score, total_score, applicable_metrics = calculate_vulnerability_resolution_time(time_dict, app_id, total_score, applicable_metrics) + # calculate frequency of security scans + scan_score, total_score, applicable_metrics = calculate_frequency_of_security_scans(time_dict, app_id, total_score, applicable_metrics) + # calculate mean time to detect (MTTD) security issues + all_vulns, detection_score, total_score, applicable_metrics = calculate_mean_time_to_detect(time_dict, app_id, total_score, applicable_metrics) + # calculate percentage of critical/high risk issues addressed + high_severity_score, total_score, applicable_metrics = calculate_percentage_of_high_risk_issues_addressed(all_vulns, total_score, applicable_metrics) + # calculate compliance with security standards + compliance_score, total_score, applicable_metrics = calculate_compliance_with_security_standards(time_dict, app_id, total_score, applicable_metrics) + # calculate risk profile adherence + risk_score, total_score, applicable_metrics = calculate_risk_profile_adherence(app_id, total_score, applicable_metrics) + + # calculate grade and final score + max_score, percent_score, grade = _calculate_grade(applicable_metrics, total_score) + + score = { + 'grade': grade, + 'total_score': total_score, + 'percent_score': (percent_score * 100), + 'max_score': max_score, + 'vulnerability_remediation_score': vr_score, + 'scan_frequency_score': scan_score, + 'time_to_detect_issues_score': detection_score, + 'critical_and_high_risk_mitigation_score': high_severity_score, + 'compliance_with_security_standards_score': compliance_score, + 'risk_profile_adherence_score': risk_score + } + response = jsonify(score), 200 + return response + + +def calculate_vulnerability_resolution_time(time_dict, app_id, total_score, applicable_metrics): + if '--' in app_id: + app_name = app_id.split('--')[0] + app_component = app_id.split('--')[1] + resolved_vulns = Vulnerabilities.query \ + .join(BusinessApplications, BusinessApplications.ID == Vulnerabilities.ApplicationId) \ + .filter(BusinessApplications.ApplicationName == app_name) \ + .filter(BusinessApplications.ApplicationAcronym == app_component) \ + .filter(Vulnerabilities.Status == 'Closed-Mitigated') \ + .filter(Vulnerabilities.MitigationDate >= time_dict['timeStart']) \ + .filter(Vulnerabilities.MitigationDate <= time_dict['timeEnd']) \ + .all() + else: + resolved_vulns = Vulnerabilities.query \ + .join(BusinessApplications, BusinessApplications.ID == Vulnerabilities.ApplicationId) \ + .filter(BusinessApplications.ApplicationName == app_id) \ + .filter(Vulnerabilities.Status == 'Closed-Mitigated') \ + .filter(Vulnerabilities.MitigationDate >= time_dict['timeStart']) \ + .filter(Vulnerabilities.MitigationDate <= time_dict['timeEnd']) \ + .all() + total_time = 0 + for i in resolved_vulns: + add_date = i.AddDate + resolve_date = i.MitigationDate + total = resolve_date - add_date + total_time += total.seconds / 60 / 60 + ave_resolved_hours = total_time / len(resolved_vulns) + vr_score = _calculate_dora_metric_score('resolved_vulns', ave_resolved_hours) + total_score += vr_score + applicable_metrics += 1 + return vr_score, total_score, applicable_metrics + + +def calculate_frequency_of_security_scans(time_dict, app_id, total_score, applicable_metrics): + if '--' in app_id: + app_name = app_id.split('--')[0] + app_component = app_id.split('--')[1] + all_scans = VulnerabilityScans.query \ + .join(BusinessApplications, BusinessApplications.ID == VulnerabilityScans.ApplicationId) \ + .filter(BusinessApplications.ApplicationName == app_name) \ + .filter(BusinessApplications.ApplicationAcronym == app_component) \ + .filter(VulnerabilityScans.ScanStartDate >= time_dict['timeStart']) \ + .filter(VulnerabilityScans.ScanStartDate <= time_dict['timeEnd']) \ + .all() + else: + all_scans = VulnerabilityScans.query \ + .join(BusinessApplications, BusinessApplications.ID == VulnerabilityScans.ApplicationId) \ + .filter(BusinessApplications.ApplicationName == app_id) \ + .filter(VulnerabilityScans.ScanStartDate >= time_dict['timeStart']) \ + .filter(VulnerabilityScans.ScanStartDate <= time_dict['timeEnd']) \ + .all() + if '--' in app_id: + app_name = app_id.split('--')[0] + app_component = app_id.split('--')[1] + reg_date = BusinessApplications\ + .query\ + .with_entities(BusinessApplications.RegDate)\ + .filter(BusinessApplications.ApplicationName == app_name) \ + .filter(BusinessApplications.ApplicationAcronym == app_component) \ + .first()[0] + else: + reg_date = BusinessApplications.query.with_entities(BusinessApplications.RegDate).filter( + BusinessApplications.ApplicationName == app_id).first()[0] + now = datetime.datetime.utcnow() + total_duration = (now - reg_date).days + unique_scan_dates = set() + for scan in all_scans: + unique_scan_dates.add(scan.ScanStartDate.date()) + unique_scan_count = len(unique_scan_dates) + if total_duration > 0: + frequency_of_scans = unique_scan_count / total_duration + else: + frequency_of_scans = 0 + scan_score = _calculate_dora_metric_score('scan_frequency', frequency_of_scans) + total_score += scan_score + applicable_metrics += 1 + return scan_score, total_score, applicable_metrics + + +def calculate_mean_time_to_detect(time_dict, app_id, total_score, applicable_metrics): + if '--' in app_id: + app_name = app_id.split('--')[0] + app_component = app_id.split('--')[1] + all_vulns = Vulnerabilities.query \ + .join(BusinessApplications, BusinessApplications.ID == Vulnerabilities.ApplicationId) \ + .filter(BusinessApplications.ApplicationName == app_name) \ + .filter(BusinessApplications.ApplicationAcronym == app_component) \ + .filter(Vulnerabilities.AddDate >= time_dict['timeStart']) \ + .filter(Vulnerabilities.AddDate <= time_dict['timeEnd']) \ + .all() + else: + all_vulns = Vulnerabilities.query \ + .join(BusinessApplications, BusinessApplications.ID == Vulnerabilities.ApplicationId) \ + .filter(BusinessApplications.ApplicationName == app_id) \ + .filter(Vulnerabilities.AddDate >= time_dict['timeStart']) \ + .filter(Vulnerabilities.AddDate <= time_dict['timeEnd']) \ + .all() + total_duration = 0 + for i in all_vulns: + detection_date = i.ReleaseDate + identification_date = i.AddDate + detection_time = detection_date - identification_date + total_duration += detection_time.seconds + if total_duration > 0: + detection_time_ave = total_duration / len(all_vulns) + else: + detection_time_ave = 0 + detection_score = _calculate_dora_metric_score('detection_time', detection_time_ave) + total_score += detection_score + applicable_metrics += 1 + return all_vulns, detection_score, total_score, applicable_metrics + + +def calculate_percentage_of_high_risk_issues_addressed(all_vulns, total_score, applicable_metrics): + total_issues = 0 + addressed_issues = 0 + for i in all_vulns: + if i.Severity == 'Critical' or i.Severity == 'High': + total_issues += 1 + if i.Status == 'Closed-Mitigated': + addressed_issues += 1 + if addressed_issues: + percent_addressed = (addressed_issues / total_issues) * 100 + else: + percent_addressed = 0 + high_severity_score = _calculate_dora_metric_score('high_severity', percent_addressed) + total_score += high_severity_score + applicable_metrics += 1 + return high_severity_score, total_score, applicable_metrics + + +def calculate_compliance_with_security_standards(time_dict, app_id, total_score, applicable_metrics): + if '--' in app_id: + app_name = app_id.split('--')[0] + app_component = app_id.split('--')[1] + latest_assessments = AssessmentBenchmarkAssessments.query \ + .join(BusinessApplications, BusinessApplications.ID == AssessmentBenchmarkAssessments.ApplicationID) \ + .filter(BusinessApplications.ApplicationName == app_name) \ + .filter(BusinessApplications.ApplicationAcronym == app_component) \ + .order_by(AssessmentBenchmarkAssessments.BenchmarkID, + desc(AssessmentBenchmarkAssessments.AddDate)) \ + .distinct(AssessmentBenchmarkAssessments.BenchmarkID) \ + .filter(AssessmentBenchmarkAssessments.AddDate >= time_dict['timeStart']) \ + .filter(AssessmentBenchmarkAssessments.AddDate <= time_dict['timeEnd']) \ + .all() + else: + latest_assessments = AssessmentBenchmarkAssessments.query \ + .join(BusinessApplications, BusinessApplications.ID == AssessmentBenchmarkAssessments.ApplicationID) \ + .filter(BusinessApplications.ApplicationName == app_id) \ + .order_by(AssessmentBenchmarkAssessments.BenchmarkID, + desc(AssessmentBenchmarkAssessments.AddDate)) \ + .distinct(AssessmentBenchmarkAssessments.BenchmarkID) \ + .filter(AssessmentBenchmarkAssessments.AddDate >= time_dict['timeStart']) \ + .filter(AssessmentBenchmarkAssessments.AddDate <= time_dict['timeEnd']) \ + .all() + all_rules = 0 + all_passed = 0 + for i in latest_assessments: + assessment_rules = AssessmentBenchmarkRules \ + .query \ + .filter(AssessmentBenchmarkRules.BenchmarkID == i.BenchmarkID) \ + .filter(text('AssessmentBenchmarkRules.ImplementationLevels LIKE "%1%"')) \ + .all() + assessment_rules_passed = AssessmentBenchmarkRuleAudits \ + .query \ + .filter(AssessmentBenchmarkRuleAudits.AssessmentID == i.ID) \ + .filter(text('AssessmentBenchmarkRuleAudits.PassingLevels LIKE "%1%"')) \ + .all() + all_rules += len(assessment_rules) + all_passed += len(assessment_rules_passed) + if all_passed > 0: + benchmark_pass_percent = all_passed / all_rules + else: + benchmark_pass_percent = 0 + compliance_score = _calculate_dora_metric_score('compliance', benchmark_pass_percent) + total_score += compliance_score + applicable_metrics += 1 + return compliance_score, total_score, applicable_metrics + + +def calculate_risk_profile_adherence(app_id, total_score, applicable_metrics): + if '--' in app_id: + app_name = app_id.split('--')[0] + app_component = app_id.split('--')[1] + score_query = BusinessApplications \ + .query \ + .with_entities(BusinessApplications.Criticality) \ + .filter(BusinessApplications.ApplicationName == app_name) \ + .filter(BusinessApplications.ApplicationAcronym == app_component) \ + .filter(text('BusinessApplications.Criticality LIKE "%(%"')) \ + .first() + else: + score_query = BusinessApplications \ + .query \ + .with_entities(BusinessApplications.Criticality) \ + .filter(BusinessApplications.ApplicationName == app_id) \ + .filter(text('BusinessApplications.Criticality LIKE "%(%"')) \ + .first() + if score_query is not None: + score = score_query.Criticality.split()[0] + else: + score = 0 + risk_score = _calculate_dora_metric_score('risk', score) + total_score += risk_score + applicable_metrics += 1 + return risk_score, total_score, applicable_metrics + + +def _calculate_grade(applicable_metrics, total_score): + max_score = applicable_metrics * 5 + percent_score = total_score / max_score + if percent_score >= 0.97: + grade = 'A+' + elif percent_score >= 0.93 and percent_score < 0.97: + grade = 'A' + elif percent_score >= 0.90 and percent_score < 0.93: + grade = 'A-' + elif percent_score >= 0.87 and percent_score < 0.90: + grade = 'B+' + elif percent_score >= 0.83 and percent_score < 0.87: + grade = 'B' + elif percent_score >= 0.80 and percent_score < 0.83: + grade = 'B-' + elif percent_score >= 0.77 and percent_score < 0.80: + grade = 'C+' + elif percent_score >= 0.73 and percent_score < 0.77: + grade = 'C' + elif percent_score >= 0.70 and percent_score < 0.73: + grade = 'C-' + elif percent_score >= 0.67 and percent_score < 0.70: + grade = 'D+' + elif percent_score >= 0.63 and percent_score < 0.67: + grade = 'D' + elif percent_score >= 0.60 and percent_score < 0.63: + grade = 'D-' + else: + grade = 'F' + return max_score, percent_score, grade + +def _calculate_dora_metric_score(metric_type, metric_data): + if metric_type == 'resolved_vulns': + if metric_data < 24: + points = 5 + elif metric_data >= 24 and metric_data < 72: + points = 4 + elif metric_data >= 72 and metric_data < 168: + points = 3 + elif metric_data: + points = 1 + else: + points = 0 + elif metric_type == 'scan_frequency': + if metric_data == 1: # means the scans occurred daily + points = 5 + elif metric_data > 0.1428571428571429: # means scans occurred at least weekly + points = 4 + elif metric_data > 0.07142857142857143: # means scans occurred at least monthly + points = 3 + elif metric_data: # means scans occurred less than monthly + points = 1 + else: # means no scans have be conducted + points = 0 + elif metric_type == 'detection_time': + if metric_data < 3600: # means less than 1 hour + points = 5 + elif metric_data < 21600: # means less than 6 hours + points = 4 + elif metric_data < 86400: # means less than 24 hours + points = 3 + elif metric_data: # means more than 24 hours + points = 1 + else: # means no scans have be conducted + points = 0 + elif metric_type == 'high_severity': + if metric_data > 95: + points = 5 + elif metric_data >= 80 and metric_data <= 95: + points = 4 + elif metric_data >= 60 and metric_data < 80: + points = 3 + elif metric_data: + points = 1 + else: # means no scans have be conducted + points = 0 + elif metric_type == 'compliance': + if metric_data > 95: + points = 5 + elif metric_data >= 80 and metric_data <= 95: + points = 4 + elif metric_data >= 60 and metric_data < 80: + points = 3 + elif metric_data: + points = 1 + else: # means no scans have be conducted + points = 0 + elif metric_type == 'risk': + if metric_data == 'low': + points = 5 + elif metric_data == 'medium': + points = 4 + elif metric_data == 'high': + points = 3 + else: # means no scans have be conducted + points = 0 + return points + diff --git a/src/vr/db_models/setup.py b/src/vr/db_models/setup.py index 31160bba..67adfc3b 100644 --- a/src/vr/db_models/setup.py +++ b/src/vr/db_models/setup.py @@ -34,7 +34,7 @@ def _init_db(db=None, app=None): class User(UserMixin, db.Model): __tablename__ = 'User' - extend_existing = True + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer, primary_key=True) is_active = db.Column('is_active', db.Boolean(), nullable=False, server_default='1') is_admin = db.Column('is_admin', db.Boolean(), nullable=False, server_default='0') @@ -78,6 +78,7 @@ def __repr__(self): # Define the Role data-model class UserRoles(db.Model): __tablename__ = 'UserRoles' + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer(), primary_key=True) name = db.Column(db.String(50), unique=True) description = db.Column(db.String(200)) @@ -87,6 +88,7 @@ class UserRoles(db.Model): # Define the UserRoles association table class UserRoleAssignments(db.Model): __tablename__ = 'UserRoleAssignments' + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer(), primary_key=True) user_id = db.Column(db.Integer(), db.ForeignKey(USER_ID, ondelete='CASCADE')) role_id = db.Column(db.Integer(), db.ForeignKey('UserRoles.id', ondelete='CASCADE')) @@ -99,6 +101,7 @@ class UserRoleAssignments(db.Model): class EntityPermissions(db.Model): __tablename__ = 'EntityPermissions' + __table_args__ = {'extend_existing': True} ID = db.Column(db.Integer, primary_key=True) AddDate = db.Column(db.DateTime, index=True, default=datetime.utcnow, nullable=False) UserID = db.Column(db.Integer, db.ForeignKey(USER_ID, ondelete='CASCADE')) @@ -111,6 +114,7 @@ class EntityPermissions(db.Model): class AppConfig(db.Model): __tablename__ = 'AppConfig' + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer, primary_key=True) first_access = db.Column(db.Boolean, nullable=False, default=True) diff --git a/src/vr/db_models/setup_2.py b/src/vr/db_models/setup_2.py index 605d54ab..d47ba210 100644 --- a/src/vr/db_models/setup_2.py +++ b/src/vr/db_models/setup_2.py @@ -47,7 +47,7 @@ class Technologies(db.Model): UniqueIDType = db.Column(db.String(20)) Description = db.Column(db.String(200)) RegComplete = db.Column(db.String(1)) - RegDate = db.Column(db.DateTime, index=True, default=datetime.utcnow, nullable=False) + RegDate = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) Technologies() diff --git a/src/vr/templates/admin/settings.html b/src/vr/templates/admin/settings.html index b1d00a7a..06a4bee5 100644 --- a/src/vr/templates/admin/settings.html +++ b/src/vr/templates/admin/settings.html @@ -1,7 +1,48 @@ {% extends 'base_auth.html' %} {% block app_content %} + + + + {{ super() }} @@ -14,7 +55,61 @@

Application Settings

From fb5d50f2c373191b9f6288c61aa5ec222f4b02d5 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 8 Jan 2024 08:11:48 -0800 Subject: [PATCH 09/51] Feature/jenkinsfile updates (#455) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bb221109..7d21f73e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,7 +51,7 @@ pipeline { stage('Unit Testing') { agent { docker { - image 'securityuniversal/jenkins-python-agent:latest' + image 'securityuniversal/jenkins:latest' } } when { From 22fc7571dc96690bc060d156429e30a1d8a819d2 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:26:18 -0800 Subject: [PATCH 10/51] Feature/jenkinsfile updates (#457) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7d21f73e..1fffa348 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,7 +99,7 @@ pipeline { stage('Software Composition Analysis') { agent { docker { - image 'securityuniversal/jenkins-codetesting-agent:latest' + image 'securityuniversal/jenkins:latest' } } when { @@ -127,7 +127,7 @@ pipeline { stage('Static Application Security Testing') { agent { docker { - image 'securityuniversal/jenkins-codetesting-agent:latest' + image 'securityuniversal/jenkins:latest' } } when { From 48869b85b94bc0b142bd039944cf979fe789bff8 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 8 Jan 2024 14:36:05 -0800 Subject: [PATCH 11/51] Feature/jenkinsfile updates (#459) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml --- pipeline-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 9d98c47f..68f8f7cf 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -23,7 +23,7 @@ stages: - Python - Javascript sast: - enabled: true + enabled: false branches: - release codeLanguages: From b88f33e01e91cb761d98b04519d69cd1399d44c9 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:07:07 -0800 Subject: [PATCH 12/51] Feature/jenkinsfile updates (#461) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml --- pipeline-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 68f8f7cf..2b33b743 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -9,14 +9,14 @@ stages: branches: - release unitTesting: - enabled: true + enabled: false branches: [] secretScanning: enabled: true branches: - release sca: - enabled: true + enabled: false branches: - release codeLanguages: @@ -37,19 +37,19 @@ stages: branches: - release containerScan: - enabled: true + enabled: false branches: - release containerName: secusphere containerTag: latest releaseToTest: - enabled: true + enabled: false branches: - release serviceName: secusphere containerTag: latest testRelease: - enabled: true + enabled: false branches: - release targetUrl: 'http://192.168.0.68:5010' From 3058cc8623e3fb3578035d300765d8ffc54d6a44 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:13:50 -0800 Subject: [PATCH 13/51] Feature/jenkinsfile updates (#463) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml --- pipeline-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 2b33b743..c54451b7 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -56,7 +56,7 @@ stages: dastTestType: full apiTargetUrl: 'http://192.168.0.68:5010/api/openapi.yaml' securityQualityGate: - enabled: true + enabled: false branches: - release deploy: From eae9cfd046a1800d8d80fe8e2f4edabc68687900 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:25:23 -0800 Subject: [PATCH 14/51] Feature/jenkinsfile updates (#465) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 1fffa348..a0e0cd5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -316,6 +316,7 @@ pipeline { agent { docker { image 'securityuniversal/jenkins-deploy-agent:latest' + args '--group-add 999' } } when { From b1d358668fdd463e8262ee93eb88ed6649fcb98e Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:59:29 -0800 Subject: [PATCH 15/51] Feature/jenkinsfile updates (#467) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py --- src/vr/vulns/model/sgglobalthresholds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vr/vulns/model/sgglobalthresholds.py b/src/vr/vulns/model/sgglobalthresholds.py index 0309bae3..09a2d3f1 100644 --- a/src/vr/vulns/model/sgglobalthresholds.py +++ b/src/vr/vulns/model/sgglobalthresholds.py @@ -8,7 +8,7 @@ class SgGlobalThresholds(db.Model): __tablename__ = 'SgGlobalThresholds' __table_args__ = {'extend_existing': True} ID = db.Column(db.Integer, primary_key=True) - Name = db.Column(db.String) + Name = db.Column(db.String(100)) AddDate = db.Column(db.DateTime) ThreshScaLow = db.Column(db.Integer) ThreshScaMedium = db.Column(db.Integer) From a688f721349a8cc8c330319f434c74ce6b0cbd69 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:21:14 -0800 Subject: [PATCH 16/51] Feature/jenkinsfile updates (#469) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py --- src/vr/functions/initial_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vr/functions/initial_setup.py b/src/vr/functions/initial_setup.py index abc1228d..1132f9c9 100644 --- a/src/vr/functions/initial_setup.py +++ b/src/vr/functions/initial_setup.py @@ -217,7 +217,7 @@ def setup_core_db_tables(ENV): if ENV == 'test': sql = 'INSERT INTO SgGlobalThresholds (Name, AddDate, ThreshScaLow, ThreshScaMedium, ThreshScaHigh, ThreshScaCritical, ThreshContainerLow, ThreshContainerMedium, ThreshContainerHigh, ThreshContainerCritical, ThreshDastLow, ThreshDastMedium, ThreshDastHigh, ThreshDastCritical, ThreshDastApiLow, ThreshDastApiMedium, ThreshDastApiHigh, ThreshDastApiCritical, ThreshInfrastructureLow, ThreshInfrastructureMedium, ThreshInfrastructureHigh, ThreshInfrastructureCritical, ThreshSastLow, ThreshSastMedium, ThreshSastHigh, ThreshSastCritical, ThreshIacLow, ThreshIacMedium, ThreshIacHigh, ThreshIacCritical, ThreshSecretsLow, ThreshSecretsMedium, ThreshSecretsHigh, ThreshSecretsCritical) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)' else: - sql = 'INSERT INTO SgGlobalThresholds (Name, AddDate, ThreshScaLow, ThreshScaMedium, ThreshScaHigh, ThreshScaCritical, ThreshContainerLow, ThreshContainerMedium, ThreshContainerHigh, ThreshContainerCritical, ThreshDastLow, ThreshDastMedium, ThreshDastHigh, ThreshDastCritical, ThreshDastApiLow, ThreshDastApiMedium, ThreshDastApiHigh, ThreshDastApiCritical, ThreshInfrastructureLow, ThreshInfrastructureMedium, ThreshInfrastructureHigh, ThreshInfrastructureCritical, ThreshSastLow, ThreshSastMedium, ThreshSastHigh, ThreshSastCritical, ThreshIacLow, ThreshIacMedium, ThreshIacHigh, ThreshIacCritical, ThreshSecretsLow, ThreshSecretsMedium, ThreshSecretsHigh, ThreshSecretsCritical) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)' + sql = 'INSERT INTO SgGlobalThresholds (Name, AddDate, ThreshScaLow, ThreshScaMedium, ThreshScaHigh, ThreshScaCritical, ThreshContainerLow, ThreshContainerMedium, ThreshContainerHigh, ThreshContainerCritical, ThreshDastLow, ThreshDastMedium, ThreshDastHigh, ThreshDastCritical, ThreshDastApiLow, ThreshDastApiMedium, ThreshDastApiHigh, ThreshDastApiCritical, ThreshInfrastructureLow, ThreshInfrastructureMedium, ThreshInfrastructureHigh, ThreshInfrastructureCritical, ThreshSastLow, ThreshSastMedium, ThreshSastHigh, ThreshSastCritical, ThreshIacLow, ThreshIacMedium, ThreshIacHigh, ThreshIacCritical, ThreshSecretsLow, ThreshSecretsMedium, ThreshSecretsHigh, ThreshSecretsCritical) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)' args = ("General", now, None,None,4,0,None,None,2,0,None,None,0,2,None,None,0,2,None,None,0,2,None,None,0,2,None,None,0,2,None,None,0,2) cur.execute(sql, args) db.commit() From d0edd8f43198c3e36fe21e092a87b05a5679668a Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 12 Jan 2024 20:34:28 -0800 Subject: [PATCH 17/51] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a0e0cd5d..d2593530 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -326,7 +326,7 @@ pipeline { // Condition for a Test-* branch expression { // Split the branch name by '/' and check if the last segment starts with 'Test-' - env.BRANCH_NAME.split('/').last().startsWith('Test') + env.BRANCH_NAME.split('/').last().startsWith('staging') } } } From 6bc2607e607869e15e8ed4e2814f5428b1a7ccc1 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 12 Jan 2024 22:06:35 -0800 Subject: [PATCH 18/51] Update pipeline-config.yaml --- pipeline-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index c54451b7..2267ddc6 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -75,6 +75,7 @@ stages: azure.azTenantId: 'azTenantId' serviceCredentials: {} serviceSetStrings: + app.env: test app.extUrl: "192.168.0.150" app.db.prodDbUriRef: "PROD-DB-URI" app.smtp.host: "smtp.sendgrid.net:587" From 4e2bc4af7db96e100b2dfa52480a626b9046cabb Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 12 Jan 2024 22:22:47 -0800 Subject: [PATCH 19/51] Update values.yaml --- ci_cd/helm/secusphere/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_cd/helm/secusphere/values.yaml b/ci_cd/helm/secusphere/values.yaml index 8f2bcf7d..abba5342 100644 --- a/ci_cd/helm/secusphere/values.yaml +++ b/ci_cd/helm/secusphere/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. environment: prod appName: "secusphere" -appDomain: "acme.com" +appDomain: "securityuniversal.com" tlsSecretName: su-wildcard-tls From fba2f2bcd1453922682cc605c3e0387e43d12dfe Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 12 Jan 2024 23:10:34 -0800 Subject: [PATCH 20/51] Update values.yaml --- ci_cd/helm/su-secrets/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_cd/helm/su-secrets/values.yaml b/ci_cd/helm/su-secrets/values.yaml index 3bba9d33..97c59f40 100644 --- a/ci_cd/helm/su-secrets/values.yaml +++ b/ci_cd/helm/su-secrets/values.yaml @@ -8,6 +8,6 @@ azure: tls: enabled: true - name: "dynamic" + name: "su-wildcard-tls" crt: "dynamic" key: "dynamic" From 1a298c05ea84e93dc9eda4314e6796a6e11abe7b Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:13:47 -0800 Subject: [PATCH 21/51] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d2593530..b45f5c3d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -326,7 +326,7 @@ pipeline { // Condition for a Test-* branch expression { // Split the branch name by '/' and check if the last segment starts with 'Test-' - env.BRANCH_NAME.split('/').last().startsWith('staging') + env.BRANCH_NAME.split('/').last().startsWith('staging') || env.BRANCH_NAME.split('/').last().startsWith('Prod') } } } From ef00ba59f8b78977252d313d3a83725d9c4e852d Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 15 Jan 2024 08:39:41 -0800 Subject: [PATCH 22/51] Update values.yaml --- ci_cd/helm/secusphere/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_cd/helm/secusphere/values.yaml b/ci_cd/helm/secusphere/values.yaml index abba5342..8a4fce89 100644 --- a/ci_cd/helm/secusphere/values.yaml +++ b/ci_cd/helm/secusphere/values.yaml @@ -83,7 +83,7 @@ volumes: claimName: su-webapp-pv-claim ingress: - enabled: true + enabled: false serviceAccount: # Specifies whether a service account should be created From f497be3383606476ced847f9fef26186d372ab51 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:19:36 -0800 Subject: [PATCH 23/51] Update pipeline-config.yaml --- pipeline-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 2267ddc6..daaeead8 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -75,14 +75,14 @@ stages: azure.azTenantId: 'azTenantId' serviceCredentials: {} serviceSetStrings: - app.env: test - app.extUrl: "192.168.0.150" + app.env: prod + app.extUrl: "secusphere.securityuniversal.com" app.db.prodDbUriRef: "PROD-DB-URI" app.smtp.host: "smtp.sendgrid.net:587" app.smtp.user: apikey app.smtp.adminEmail: "admin@securityuniversal.com" - app.smtp.passwordRef: "SENDGRID-SMTP-PW" - app.az.keyVaultName: "BkDevSecOpsKeyVault" + app.smtp.passwordRef: "SMTP-PW" + app.az.keyVaultName: "ss-keyvault" post: enabled: true branches: From ee6d9dd0bfb3c7ffd9ad26ff43ee391f3443ced6 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 16 Jan 2024 20:29:15 -0800 Subject: [PATCH 24/51] Feature/jenkinsfile updates (#473) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations --- src/config_engine.py | 130 +++++++++++++++++++------------- src/settings.py | 2 + src/vr/templates/base_auth.html | 1 + 3 files changed, 79 insertions(+), 54 deletions(-) diff --git a/src/config_engine.py b/src/config_engine.py index 0168cb9a..07400d06 100644 --- a/src/config_engine.py +++ b/src/config_engine.py @@ -9,7 +9,7 @@ SET_APP_EXT_URL from settings import SET_PROD_DB_URI_REF, SET_SMTP_PW_REF, SET_JENKINS_KEY_REF, SET_JENKINS_USER_REF, SET_JENKINS_TOKEN_REF from settings import SET_PROD_DB_URI, SET_SMTP_PW, SET_JENKINS_KEY, SET_JENKINS_USER, \ - SET_JENKINS_HOST, SET_JENKINS_PROJECT, SET_JENKINS_TOKEN, SET_JENKINS_STAGING_PROJECT + SET_JENKINS_HOST, SET_JENKINS_PROJECT, SET_JENKINS_TOKEN, SET_JENKINS_STAGING_PROJECT, SET_JENKINS_ENABLED, SET_SNOW_ENABLED from settings import SET_AZAD_CLIENT_ID, SET_AZAD_CLIENT_SECRET, SET_AZAD_AUTHORITY from settings import SET_SNOW_INSTANCE_NAME, SET_SNOW_CLIENT_ID, SET_SNOW_CLIENT_SECRET, SET_SNOW_USERNAME, SET_SNOW_PASSWORD, SET_SNOW_CLIENT_SECRET_REF, SET_SNOW_PASSWORD_REF @@ -205,65 +205,87 @@ def delete_cert(self, secret_name): ## ## GitHub to Jenkins Webhook ## -if ENV == 'prod': - if os.getenv('JENKINS_USER'): - JENKINS_USER = KeyVaultManager().get_secret(os.getenv('JENKINS_USER')) - else: - JENKINS_USER = KeyVaultManager().get_secret(SET_JENKINS_USER_REF) - if os.getenv('JENKINS_KEY'): - JENKINS_KEY = KeyVaultManager().get_secret(os.getenv('JENKINS_KEY')) - else: - JENKINS_KEY = KeyVaultManager().get_secret(SET_JENKINS_KEY_REF) - if os.getenv('JENKINS_TOKEN'): - JENKINS_TOKEN = KeyVaultManager().get_secret(os.getenv('JENKINS_TOKEN')) - else: - JENKINS_TOKEN = KeyVaultManager().get_secret(SET_JENKINS_TOKEN_REF) +if os.getenv('JENKINS_ENABLED'): + JENKINS_ENABLED = os.getenv('JENKINS_ENABLED') else: - JENKINS_USER = SET_JENKINS_USER - JENKINS_KEY = SET_JENKINS_KEY - JENKINS_TOKEN = SET_JENKINS_TOKEN + JENKINS_ENABLED = SET_JENKINS_ENABLED +if JENKINS_ENABLED == 'yes': + if ENV == 'prod': + if os.getenv('JENKINS_USER'): + JENKINS_USER = KeyVaultManager().get_secret(os.getenv('JENKINS_USER')) + else: + JENKINS_USER = KeyVaultManager().get_secret(SET_JENKINS_USER_REF) + if os.getenv('JENKINS_KEY'): + JENKINS_KEY = KeyVaultManager().get_secret(os.getenv('JENKINS_KEY')) + else: + JENKINS_KEY = KeyVaultManager().get_secret(SET_JENKINS_KEY_REF) + if os.getenv('JENKINS_TOKEN'): + JENKINS_TOKEN = KeyVaultManager().get_secret(os.getenv('JENKINS_TOKEN')) + else: + JENKINS_TOKEN = KeyVaultManager().get_secret(SET_JENKINS_TOKEN_REF) + else: + JENKINS_USER = SET_JENKINS_USER + JENKINS_KEY = SET_JENKINS_KEY + JENKINS_TOKEN = SET_JENKINS_TOKEN -if os.getenv('JENKINS_PROJECT'): - JENKINS_PROJECT = os.getenv('JENKINS_PROJECT') -else: - JENKINS_PROJECT = SET_JENKINS_PROJECT + if os.getenv('JENKINS_PROJECT'): + JENKINS_PROJECT = os.getenv('JENKINS_PROJECT') + else: + JENKINS_PROJECT = SET_JENKINS_PROJECT -if os.getenv('JENKINS_HOST'): - JENKINS_HOST = os.getenv('JENKINS_HOST') -else: - JENKINS_HOST = SET_JENKINS_HOST + if os.getenv('JENKINS_HOST'): + JENKINS_HOST = os.getenv('JENKINS_HOST') + else: + JENKINS_HOST = SET_JENKINS_HOST -if os.getenv('JENKINS_STAGING_PROJECT'): - JENKINS_STAGING_PROJECT = os.getenv('JENKINS_STAGING_PROJECT') + if os.getenv('JENKINS_STAGING_PROJECT'): + JENKINS_STAGING_PROJECT = os.getenv('JENKINS_STAGING_PROJECT') + else: + JENKINS_STAGING_PROJECT = SET_JENKINS_STAGING_PROJECT else: - JENKINS_STAGING_PROJECT = SET_JENKINS_STAGING_PROJECT - + JENKINS_USER = "" + JENKINS_KEY = "" + JENKINS_TOKEN = "" + JENKINS_PROJECT = "" + JENKINS_HOST = "" + JENKINS_STAGING_PROJECT = "" ## ServiceNOW Integration -if ENV == 'prod': - if os.getenv('SNOW_PASSWORD'): - SNOW_PASSWORD = KeyVaultManager().get_secret(os.getenv('SNOW_PASSWORD')) - else: - SNOW_PASSWORD = KeyVaultManager().get_secret(SET_SNOW_PASSWORD_REF) - if os.getenv('SNOW_CLIENT_SECRET'): - SNOW_CLIENT_SECRET = KeyVaultManager().get_secret(os.getenv('SNOW_CLIENT_SECRET')) - else: - SNOW_CLIENT_SECRET = KeyVaultManager().get_secret(SET_SNOW_CLIENT_SECRET_REF) - if os.getenv('SNOW_INSTANCE_NAME'): - SNOW_INSTANCE_NAME = KeyVaultManager().get_secret(os.getenv('SNOW_INSTANCE_NAME')) - else: - SNOW_INSTANCE_NAME = KeyVaultManager().get_secret(SET_SNOW_INSTANCE_NAME) - if os.getenv('SNOW_CLIENT_ID'): - SNOW_CLIENT_ID = KeyVaultManager().get_secret(os.getenv('SNOW_CLIENT_ID')) - else: - SNOW_CLIENT_ID = KeyVaultManager().get_secret(SET_SNOW_CLIENT_ID) - if os.getenv('SNOW_USERNAME'): - SNOW_USERNAME = KeyVaultManager().get_secret(os.getenv('SNOW_USERNAME')) +if os.getenv('SNOW_ENABLED'): + SNOW_ENABLED = os.getenv('SNOW_ENABLED') +else: + SNOW_ENABLED = SET_SNOW_ENABLED +if SNOW_ENABLED == 'yes': + if ENV == 'prod': + if os.getenv('SNOW_PASSWORD'): + SNOW_PASSWORD = KeyVaultManager().get_secret(os.getenv('SNOW_PASSWORD')) + else: + SNOW_PASSWORD = KeyVaultManager().get_secret(SET_SNOW_PASSWORD_REF) + if os.getenv('SNOW_CLIENT_SECRET'): + SNOW_CLIENT_SECRET = KeyVaultManager().get_secret(os.getenv('SNOW_CLIENT_SECRET')) + else: + SNOW_CLIENT_SECRET = KeyVaultManager().get_secret(SET_SNOW_CLIENT_SECRET_REF) + if os.getenv('SNOW_INSTANCE_NAME'): + SNOW_INSTANCE_NAME = os.getenv('SNOW_INSTANCE_NAME') + else: + SNOW_INSTANCE_NAME = SET_SNOW_INSTANCE_NAME + if os.getenv('SNOW_CLIENT_ID'): + SNOW_CLIENT_ID = os.getenv('SNOW_CLIENT_ID') + else: + SNOW_CLIENT_ID = SET_SNOW_CLIENT_ID + if os.getenv('SNOW_USERNAME'): + SNOW_USERNAME = os.getenv('SNOW_USERNAME') + else: + SNOW_USERNAME = SET_SNOW_USERNAME else: - SNOW_USERNAME = KeyVaultManager().get_secret(SET_SNOW_USERNAME) + SNOW_PASSWORD = SET_SNOW_PASSWORD + SNOW_CLIENT_SECRET = SET_SNOW_CLIENT_SECRET + SNOW_INSTANCE_NAME = SET_SNOW_INSTANCE_NAME + SNOW_CLIENT_ID = SET_SNOW_CLIENT_ID + SNOW_USERNAME = SET_SNOW_USERNAME else: - SNOW_PASSWORD = SET_SNOW_PASSWORD - SNOW_CLIENT_SECRET = SET_SNOW_CLIENT_SECRET - SNOW_INSTANCE_NAME = SET_SNOW_INSTANCE_NAME - SNOW_CLIENT_ID = SET_SNOW_CLIENT_ID - SNOW_USERNAME = SET_SNOW_USERNAME + SNOW_PASSWORD = "" + SNOW_CLIENT_SECRET = "" + SNOW_INSTANCE_NAME = "" + SNOW_CLIENT_ID = "" + SNOW_USERNAME = "" diff --git a/src/settings.py b/src/settings.py index 7273b90b..a893a2f0 100644 --- a/src/settings.py +++ b/src/settings.py @@ -39,6 +39,7 @@ ## ## Jenkins Webhook Settings (Optional) ## +SET_JENKINS_ENABLED = 'no' SET_JENKINS_KEY_REF = 'JENKINS-KEY' SET_JENKINS_USER_REF = 'JENKINS-USER' SET_JENKINS_TOKEN_REF = 'JENKINS-TOKEN' @@ -51,6 +52,7 @@ SET_JENKINS_TOKEN = 'changeme' ## ServiceNOW Settings +SET_SNOW_ENABLED = 'no' SET_SNOW_INSTANCE_NAME = 'dev124268' SET_SNOW_CLIENT_ID = '1ab21bf476013110e1ce39e1f368c2fa' SET_SNOW_CLIENT_SECRET_REF = 'SNOW-SECRET' diff --git a/src/vr/templates/base_auth.html b/src/vr/templates/base_auth.html index 5886557d..df697bc2 100644 --- a/src/vr/templates/base_auth.html +++ b/src/vr/templates/base_auth.html @@ -15,6 +15,7 @@ + From 22a872bdb968b65b6fb45010ac0d0c6fcd30a25e Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:44:25 -0800 Subject: [PATCH 25/51] Feature/jenkinsfile updates (#474) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins --- Jenkinsfile | 2 ++ src/vr/__init__.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b45f5c3d..dd45e119 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -342,6 +342,8 @@ pipeline { 'secretsSetStrings': stageConfig?.secretsSetStrings, 'serviceCredentials': stageConfig?.serviceCredentials, 'serviceSetStrings': stageConfig?.serviceSetStrings, + 'dockerReg': 'secunicontainerregistry.azurecr.io', + 'imgPullSecret': 'acrCreds' ]) } diff --git a/src/vr/__init__.py b/src/vr/__init__.py index b74f78d5..e1bc03cf 100644 --- a/src/vr/__init__.py +++ b/src/vr/__init__.py @@ -2,7 +2,7 @@ import requests from config_engine import ENV, PROD_DB_URI, AUTH_TYPE, APP_EXT_URL, LDAP_HOST, LDAP_PORT, LDAP_BASE_DN, \ LDAP_USER_DN, LDAP_GROUP_DN, LDAP_USER_RDN_ATTR, LDAP_USER_LOGIN_ATTR, LDAP_BIND_USER_DN, LDAP_BIND_USER_PASSWORD, \ - AZAD_CLIENT_ID, AZAD_CLIENT_SECRET, AZAD_AUTHORITY, JENKINS_USER + AZAD_CLIENT_ID, AZAD_CLIENT_SECRET, AZAD_AUTHORITY, JENKINS_USER, JENKINS_ENABLED from flask import Flask from flask_bootstrap import Bootstrap from flask_login import LoginManager @@ -448,4 +448,5 @@ def get_jenkins_data(): # Call the Jobs Here # train_model_every_six_hours() -get_jenkins_data_every_hour() +if JENKINS_ENABLED == 'yes': + get_jenkins_data_every_hour() From 11694577abc210f1e409b4018451a66a2265c2b0 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:18:42 -0800 Subject: [PATCH 26/51] Feature/jenkinsfile updates (#476) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins * Update Jenkinsfile --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dd45e119..52b47457 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -199,7 +199,8 @@ pipeline { jslStageWrapper('Build Docker Service') { script { jslBuildDocker([ - 'serviceName': env.appName + 'serviceName': env.appName, + 'dockerReg': 'secunicontainerregistry.azurecr.io' ]) } } @@ -230,7 +231,7 @@ pipeline { def stageConfig = jslReadYamlConfig('containerScan') def containerName = stageConfig?.containerName def containerTag = stageConfig?.containerTag - jslContainerSecurityScanning(containerName, containerTag) + jslContainerSecurityScanning(containerName, containerTag, 'secunicontainerregistry.azurecr.io') } } } From 4b6af0b1b7d45088ba5add56aeb392a83ec7ccdb Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 27 Feb 2024 20:22:24 -0800 Subject: [PATCH 27/51] Feature/jenkinsfile updates (#478) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins * Update Jenkinsfile * Update pipeline-config.yaml From cebffc6cb0dbc6f5a7ac0190cef8756506297c0b Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 27 Feb 2024 20:29:25 -0800 Subject: [PATCH 28/51] Update pipeline-config.yaml --- pipeline-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index daaeead8..55659e84 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -56,7 +56,7 @@ stages: dastTestType: full apiTargetUrl: 'http://192.168.0.68:5010/api/openapi.yaml' securityQualityGate: - enabled: false + enabled: true branches: - release deploy: From 38282ba86ef3bb91e0134a1b413c7deaf08864b0 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 27 Feb 2024 20:56:02 -0800 Subject: [PATCH 29/51] Feature/jenkinsfile updates (#481) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins * Update Jenkinsfile * Update pipeline-config.yaml * Update Jenkinsfile --- Jenkinsfile | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 52b47457..281ac45d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,7 +75,7 @@ pipeline { stage('Secret Scanning') { agent { docker { - image 'securityuniversal/jenkins-secret-agent:latest' + image 'securityuniversal/jenkins-sectesting-agent:latest' } } when { @@ -99,7 +99,7 @@ pipeline { stage('Software Composition Analysis') { agent { docker { - image 'securityuniversal/jenkins:latest' + image 'securityuniversal/jenkins-sectesting-agent:latest' } } when { @@ -127,7 +127,7 @@ pipeline { stage('Static Application Security Testing') { agent { docker { - image 'securityuniversal/jenkins:latest' + image 'securityuniversal/jenkins-sectesting-agent:latest' } } when { @@ -155,7 +155,7 @@ pipeline { stage('Infrastructure-as-Code Security Testing') { agent { docker { - image 'securityuniversal/jenkins-iac-agent:latest' + image 'securityuniversal/jenkins-sectesting-agent:latest' args '--group-add 999' } } @@ -210,7 +210,7 @@ pipeline { stage('Docker Container Scanning') { agent { docker { - image 'securityuniversal/jenkins-iac-agent:latest' + image 'securityuniversal/jenkins-sectesting-agent:latest' args '--group-add 999' } } @@ -294,6 +294,12 @@ pipeline { ////////// Quality Gate ////////// stage("Quality Gate - Security") { + agent { + docker { + image 'securityuniversal/jenkins-sectesting-agent:latest' + args '--group-add 999' + } + } when { expression { def config = jslReadYamlConfig('securityQualityGate') From 86da6a05ae9223d44dfc649d04c15c85b8599590 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 27 Feb 2024 22:07:02 -0800 Subject: [PATCH 30/51] Feature/jenkinsfile updates (#483) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins * Update Jenkinsfile * Update pipeline-config.yaml * Update Jenkinsfile * jenkins testing --- Jenkinsfile | 2 +- pipeline-config.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 281ac45d..a2d43df6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,7 +51,7 @@ pipeline { stage('Unit Testing') { agent { docker { - image 'securityuniversal/jenkins:latest' + image 'securityuniversal/jenkins-python-agent:latest' } } when { diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 55659e84..93ec1421 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -9,21 +9,21 @@ stages: branches: - release unitTesting: - enabled: false + enabled: true branches: [] secretScanning: enabled: true branches: - release sca: - enabled: false + enabled: true branches: - release codeLanguages: - Python - Javascript sast: - enabled: false + enabled: true branches: - release codeLanguages: @@ -37,19 +37,19 @@ stages: branches: - release containerScan: - enabled: false + enabled: true branches: - release containerName: secusphere containerTag: latest releaseToTest: - enabled: false + enabled: true branches: - release serviceName: secusphere containerTag: latest testRelease: - enabled: false + enabled: true branches: - release targetUrl: 'http://192.168.0.68:5010' From e019adeac8a7cd178f3b587652f3cc96a82dc81a Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 9 Mar 2024 11:32:31 -0800 Subject: [PATCH 31/51] Feature/jenkinsfile updates (#485) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins * Update Jenkinsfile * Update pipeline-config.yaml * Update Jenkinsfile * jenkins testing * Update vulnerabilities.py --- src/vr/api/vulns/vulnerabilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vr/api/vulns/vulnerabilities.py b/src/vr/api/vulns/vulnerabilities.py index 7f0c000e..68360c8b 100644 --- a/src/vr/api/vulns/vulnerabilities.py +++ b/src/vr/api/vulns/vulnerabilities.py @@ -132,7 +132,7 @@ def update_vulnerabilities_status(app_cmdb_id, scan_id, req_raw): def add_vulns_background_process(req_raw): now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") app_name = req_raw['appName'] - git_url = req_raw['giturl'] + git_url = req_raw['gitUrl'] git_branch = req_raw['branch'] findings = req_raw['findings'] scan_type = req_raw['scanType'] From 33e0b20eb87cf03926567167c608ed6dd6956960 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 9 Mar 2024 13:57:17 -0800 Subject: [PATCH 32/51] Feature/jenkinsfile updates (#487) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins * Update Jenkinsfile * Update pipeline-config.yaml * Update Jenkinsfile * jenkins testing * Update vulnerabilities.py * Update pipeline-config.yaml --- pipeline-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 93ec1421..60ce6fce 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -12,7 +12,7 @@ stages: enabled: true branches: [] secretScanning: - enabled: true + enabled: false branches: - release sca: @@ -29,7 +29,7 @@ stages: codeLanguages: - Python iac: - enabled: true + enabled: false branches: - release buildDocker: From 509a6f102f692e5756090d5e76cb8bbefb64d606 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 9 Mar 2024 14:41:24 -0800 Subject: [PATCH 33/51] Update pipeline-config.yaml --- pipeline-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 60ce6fce..2a9eb073 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -9,21 +9,21 @@ stages: branches: - release unitTesting: - enabled: true + enabled: false branches: [] secretScanning: enabled: false branches: - release sca: - enabled: true + enabled: false branches: - release codeLanguages: - Python - Javascript sast: - enabled: true + enabled: false branches: - release codeLanguages: @@ -37,26 +37,26 @@ stages: branches: - release containerScan: - enabled: true + enabled: false branches: - release containerName: secusphere containerTag: latest releaseToTest: - enabled: true + enabled: false branches: - release serviceName: secusphere containerTag: latest testRelease: - enabled: true + enabled: false branches: - release targetUrl: 'http://192.168.0.68:5010' dastTestType: full apiTargetUrl: 'http://192.168.0.68:5010/api/openapi.yaml' securityQualityGate: - enabled: true + enabled: false branches: - release deploy: From a52196c87a15f99d3970b7c5ea93adb8bbe09eba Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 9 Mar 2024 15:29:24 -0800 Subject: [PATCH 34/51] Feature/jenkinsfile updates (#490) * Update Jenkinsfile * added scores and grades api endpoint * Update Jenkinsfile * Update Jenkinsfile * Update pipeline-config.yaml * Update pipeline-config.yaml * Update pipeline-config.yaml * Update Jenkinsfile * Update sgglobalthresholds.py * Update initial_setup.py * add conditions for jenkins and snow integrations * add settings for Jenkins * Update Jenkinsfile * Update pipeline-config.yaml * Update Jenkinsfile * jenkins testing * Update vulnerabilities.py * Update pipeline-config.yaml * Update Jenkinsfile --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a2d43df6..8f818749 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -348,9 +348,7 @@ pipeline { 'secretsCredentials': stageConfig?.secretsCredentials, 'secretsSetStrings': stageConfig?.secretsSetStrings, 'serviceCredentials': stageConfig?.serviceCredentials, - 'serviceSetStrings': stageConfig?.serviceSetStrings, - 'dockerReg': 'secunicontainerregistry.azurecr.io', - 'imgPullSecret': 'acrCreds' + 'serviceSetStrings': stageConfig?.serviceSetStrings ]) } From 0f392046bbbd480c0b7936825fd51cb76cf7c4ee Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 9 Mar 2024 15:37:20 -0800 Subject: [PATCH 35/51] Update pipeline-config.yaml --- pipeline-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 60ce6fce..2a9eb073 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -9,21 +9,21 @@ stages: branches: - release unitTesting: - enabled: true + enabled: false branches: [] secretScanning: enabled: false branches: - release sca: - enabled: true + enabled: false branches: - release codeLanguages: - Python - Javascript sast: - enabled: true + enabled: false branches: - release codeLanguages: @@ -37,26 +37,26 @@ stages: branches: - release containerScan: - enabled: true + enabled: false branches: - release containerName: secusphere containerTag: latest releaseToTest: - enabled: true + enabled: false branches: - release serviceName: secusphere containerTag: latest testRelease: - enabled: true + enabled: false branches: - release targetUrl: 'http://192.168.0.68:5010' dastTestType: full apiTargetUrl: 'http://192.168.0.68:5010/api/openapi.yaml' securityQualityGate: - enabled: true + enabled: false branches: - release deploy: From ce79251e3f30ecfd7c3d00b5469ef4b7fbd64b5a Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sat, 9 Mar 2024 16:00:28 -0800 Subject: [PATCH 36/51] Update Jenkinsfile (#493) --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8f818749..bac83770 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -199,8 +199,7 @@ pipeline { jslStageWrapper('Build Docker Service') { script { jslBuildDocker([ - 'serviceName': env.appName, - 'dockerReg': 'secunicontainerregistry.azurecr.io' + 'serviceName': env.appName ]) } } @@ -231,7 +230,7 @@ pipeline { def stageConfig = jslReadYamlConfig('containerScan') def containerName = stageConfig?.containerName def containerTag = stageConfig?.containerTag - jslContainerSecurityScanning(containerName, containerTag, 'secunicontainerregistry.azurecr.io') + jslContainerSecurityScanning(containerName, containerTag) } } } From 686ae51cac2c97ecaa9ee15b58d9cf7e43d82a76 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:51:17 -0700 Subject: [PATCH 37/51] Update security_quality_gate.py (#495) --- src/vr/api/vulns/security_quality_gate.py | 128 +++++++++++----------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/src/vr/api/vulns/security_quality_gate.py b/src/vr/api/vulns/security_quality_gate.py index 930c2a1b..b00c1b1e 100644 --- a/src/vr/api/vulns/security_quality_gate.py +++ b/src/vr/api/vulns/security_quality_gate.py @@ -51,38 +51,38 @@ def _add_sg_config_settings(config, job_id, app_id): job = SgConfigSettingsPerJob( AppID=app_id, PipelineJobID = job_id, - ThreshScaLow = config['thresholds']['sca']['low'], - ThreshScaMedium = config['thresholds']['sca']['medium'], - ThreshScaHigh = config['thresholds']['sca']['high'], - ThreshScaCritical = config['thresholds']['sca']['critical'], - ThreshContainerLow = config['thresholds']['container']['low'], - ThreshContainerMedium = config['thresholds']['container']['medium'], - ThreshContainerHigh = config['thresholds']['container']['high'], - ThreshContainerCritical = config['thresholds']['container']['critical'], - ThreshDastLow = config['thresholds']['dast']['low'], - ThreshDastMedium = config['thresholds']['dast']['medium'], - ThreshDastHigh = config['thresholds']['dast']['high'], - ThreshDastCritical = config['thresholds']['dast']['critical'], - ThreshDastApiLow = config['thresholds']['dastapi']['low'], - ThreshDastApiMedium = config['thresholds']['dastapi']['medium'], - ThreshDastApiHigh = config['thresholds']['dastapi']['high'], - ThreshDastApiCritical = config['thresholds']['dastapi']['critical'], - ThreshInfrastructureLow = config['thresholds']['infrastructure']['low'], - ThreshInfrastructureMedium = config['thresholds']['infrastructure']['medium'], - ThreshInfrastructureHigh = config['thresholds']['infrastructure']['high'], - ThreshInfrastructureCritical = config['thresholds']['infrastructure']['critical'], - ThreshSastLow = config['thresholds']['sast']['low'], - ThreshSastMedium = config['thresholds']['sast']['medium'], - ThreshSastHigh = config['thresholds']['sast']['high'], - ThreshSastCritical = config['thresholds']['sast']['critical'], - ThreshIacLow = config['thresholds']['iac']['low'], - ThreshIacMedium = config['thresholds']['iac']['medium'], - ThreshIacHigh = config['thresholds']['iac']['high'], - ThreshIacCritical = config['thresholds']['iac']['critical'], - ThreshSecretsLow = config['thresholds']['secret']['low'], - ThreshSecretsMedium = config['thresholds']['secret']['medium'], - ThreshSecretsHigh = config['thresholds']['secret']['high'], - ThreshSecretsCritical = config['thresholds']['secret']['critical'], + ThreshScaLow = config['sca']['low'], + ThreshScaMedium = config['sca']['medium'], + ThreshScaHigh = config['sca']['high'], + ThreshScaCritical = config['sca']['critical'], + ThreshContainerLow = config['container']['low'], + ThreshContainerMedium = config['container']['medium'], + ThreshContainerHigh = config['container']['high'], + ThreshContainerCritical = config['container']['critical'], + ThreshDastLow = config['dast']['low'], + ThreshDastMedium = config['dast']['medium'], + ThreshDastHigh = config['dast']['high'], + ThreshDastCritical = config['dast']['critical'], + ThreshDastApiLow = config['dastapi']['low'], + ThreshDastApiMedium = config['dastapi']['medium'], + ThreshDastApiHigh = config['dastapi']['high'], + ThreshDastApiCritical = config['dastapi']['critical'], + ThreshInfrastructureLow = config['infrastructure']['low'], + ThreshInfrastructureMedium = config['infrastructure']['medium'], + ThreshInfrastructureHigh = config['infrastructure']['high'], + ThreshInfrastructureCritical = config['infrastructure']['critical'], + ThreshSastLow = config['sast']['low'], + ThreshSastMedium = config['sast']['medium'], + ThreshSastHigh = config['sast']['high'], + ThreshSastCritical = config['sast']['critical'], + ThreshIacLow = config['iac']['low'], + ThreshIacMedium = config['iac']['medium'], + ThreshIacHigh = config['iac']['high'], + ThreshIacCritical = config['iac']['critical'], + ThreshSecretsLow = config['secret']['low'], + ThreshSecretsMedium = config['secret']['medium'], + ThreshSecretsHigh = config['secret']['high'], + ThreshSecretsCritical = config['secret']['critical'], ) db.session.add(job) db_connection_handler(db) @@ -92,38 +92,38 @@ def _add_sg_results(results, job_id, app_id): job = SgResultsPerJob( AppID=app_id, PipelineJobID=job_id, - ResultScaLow=results['report']['sca']['low'], - ResultScaMedium=results['report']['sca']['medium'], - ResultScaHigh=results['report']['sca']['high'], - ResultScaCritical=results['report']['sca']['critical'], - ResultContainerLow=results['report']['container']['low'], - ResultContainerMedium=results['report']['container']['medium'], - ResultContainerHigh=results['report']['container']['high'], - ResultContainerCritical=results['report']['container']['critical'], - ResultDastLow=results['report']['dast']['low'], - ResultDastMedium=results['report']['dast']['medium'], - ResultDastHigh=results['report']['dast']['high'], - ResultDastCritical=results['report']['dast']['critical'], - ResultDastApiLow=results['report']['dastapi']['low'], - ResultDastApiMedium=results['report']['dastapi']['medium'], - ResultDastApiHigh=results['report']['dastapi']['high'], - ResultDastApiCritical=results['report']['dastapi']['critical'], - ResultInfrastructureLow=results['report']['infrastructure']['low'], - ResultInfrastructureMedium=results['report']['infrastructure']['medium'], - ResultInfrastructureHigh=results['report']['infrastructure']['high'], - ResultInfrastructureCritical=results['report']['infrastructure']['critical'], - ResultSastLow=results['report']['sast']['low'], - ResultSastMedium=results['report']['sast']['medium'], - ResultSastHigh=results['report']['sast']['high'], - ResultSastCritical=results['report']['sast']['critical'], - ResultIacLow=results['report']['iac']['low'], - ResultIacMedium=results['report']['iac']['medium'], - ResultIacHigh=results['report']['iac']['high'], - ResultIacCritical=results['report']['iac']['critical'], - ResultSecretsLow=results['report']['secret']['low'], - ResultSecretsMedium=results['report']['secret']['medium'], - ResultSecretsHigh=results['report']['secret']['high'], - ResultSecretsCritical=results['report']['secret']['critical'], + ResultScaLow=results['sca']['low'], + ResultScaMedium=results['sca']['medium'], + ResultScaHigh=results['sca']['high'], + ResultScaCritical=results['sca']['critical'], + ResultContainerLow=results['container']['low'], + ResultContainerMedium=results['container']['medium'], + ResultContainerHigh=results['container']['high'], + ResultContainerCritical=results['container']['critical'], + ResultDastLow=results['dast']['low'], + ResultDastMedium=results['dast']['medium'], + ResultDastHigh=results['dast']['high'], + ResultDastCritical=results['dast']['critical'], + ResultDastApiLow=results['dastapi']['low'], + ResultDastApiMedium=results['dastapi']['medium'], + ResultDastApiHigh=results['dastapi']['high'], + ResultDastApiCritical=results['dastapi']['critical'], + ResultInfrastructureLow=results['infrastructure']['low'], + ResultInfrastructureMedium=results['infrastructure']['medium'], + ResultInfrastructureHigh=results['infrastructure']['high'], + ResultInfrastructureCritical=results['infrastructure']['critical'], + ResultSastLow=results['sast']['low'], + ResultSastMedium=results['sast']['medium'], + ResultSastHigh=results['sast']['high'], + ResultSastCritical=results['sast']['critical'], + ResultIacLow=results['iac']['low'], + ResultIacMedium=results['iac']['medium'], + ResultIacHigh=results['iac']['high'], + ResultIacCritical=results['iac']['critical'], + ResultSecretsLow=results['secret']['low'], + ResultSecretsMedium=results['secret']['medium'], + ResultSecretsHigh=results['secret']['high'], + ResultSecretsCritical=results['secret']['critical'], ) db.session.add(job) db_connection_handler(db) From b4c8b1d574803b54f8e9552d0ebbd904d03b1f90 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:40:15 -0700 Subject: [PATCH 38/51] Feature/update jenkins config (#497) * Update security_quality_gate.py * Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bac83770..f45ebce2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,7 +21,7 @@ pipeline { env.GLOBAL_BRANCH_LIST = config.global.defaultBranches.join(',') env.CURRENT_STAGE_BRANCH_LIST = "" - jslStageWrapper.initReport() + jslStageWrapper.initReport(config) } } From 0cc02a42eb089f33dc0e9eca4dd22031771a7b4d Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 12 Mar 2024 20:22:58 -0700 Subject: [PATCH 39/51] Feature/update security gate config (#499) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py --- src/vr/api/vulns/security_quality_gate.py | 64 +++++++++++------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/vr/api/vulns/security_quality_gate.py b/src/vr/api/vulns/security_quality_gate.py index b00c1b1e..29c4191a 100644 --- a/src/vr/api/vulns/security_quality_gate.py +++ b/src/vr/api/vulns/security_quality_gate.py @@ -51,38 +51,38 @@ def _add_sg_config_settings(config, job_id, app_id): job = SgConfigSettingsPerJob( AppID=app_id, PipelineJobID = job_id, - ThreshScaLow = config['sca']['low'], - ThreshScaMedium = config['sca']['medium'], - ThreshScaHigh = config['sca']['high'], - ThreshScaCritical = config['sca']['critical'], - ThreshContainerLow = config['container']['low'], - ThreshContainerMedium = config['container']['medium'], - ThreshContainerHigh = config['container']['high'], - ThreshContainerCritical = config['container']['critical'], - ThreshDastLow = config['dast']['low'], - ThreshDastMedium = config['dast']['medium'], - ThreshDastHigh = config['dast']['high'], - ThreshDastCritical = config['dast']['critical'], - ThreshDastApiLow = config['dastapi']['low'], - ThreshDastApiMedium = config['dastapi']['medium'], - ThreshDastApiHigh = config['dastapi']['high'], - ThreshDastApiCritical = config['dastapi']['critical'], - ThreshInfrastructureLow = config['infrastructure']['low'], - ThreshInfrastructureMedium = config['infrastructure']['medium'], - ThreshInfrastructureHigh = config['infrastructure']['high'], - ThreshInfrastructureCritical = config['infrastructure']['critical'], - ThreshSastLow = config['sast']['low'], - ThreshSastMedium = config['sast']['medium'], - ThreshSastHigh = config['sast']['high'], - ThreshSastCritical = config['sast']['critical'], - ThreshIacLow = config['iac']['low'], - ThreshIacMedium = config['iac']['medium'], - ThreshIacHigh = config['iac']['high'], - ThreshIacCritical = config['iac']['critical'], - ThreshSecretsLow = config['secret']['low'], - ThreshSecretsMedium = config['secret']['medium'], - ThreshSecretsHigh = config['secret']['high'], - ThreshSecretsCritical = config['secret']['critical'], + ThreshScaLow = config['sca']['low'] if 'low' in 'sca' else '', + ThreshScaMedium = config['sca']['medium'] if 'medium' in 'sca' else '', + ThreshScaHigh = config['sca']['high'] if 'high' in 'sca' else '', + ThreshScaCritical = config['sca']['critical'] if 'critical' in 'sca' else '', + ThreshContainerLow = config['container']['low'] if 'low' in 'container' else '', + ThreshContainerMedium = config['container']['medium'] if 'medium' in 'container' else '', + ThreshContainerHigh = config['container']['high'] if 'high' in 'container' else '', + ThreshContainerCritical = config['container']['critical'] if 'critical' in 'container' else '', + ThreshDastLow = config['dast']['low'] if 'low' in 'dast' else '', + ThreshDastMedium = config['dast']['medium'] if 'medium' in 'dast' else '', + ThreshDastHigh = config['dast']['high'] if 'high' in 'dast' else '', + ThreshDastCritical = config['dast']['critical'] if 'critical' in 'dast' else '', + ThreshDastApiLow = config['dastapi']['low'] if 'low' in 'dastapi' else '', + ThreshDastApiMedium = config['dastapi']['medium'] if 'medium' in 'dastapi' else '', + ThreshDastApiHigh = config['dastapi']['high'] if 'high' in 'dastapi' else '', + ThreshDastApiCritical = config['dastapi']['critical'] if 'critical' in 'dastapi' else '', + ThreshInfrastructureLow = config['infrastructure']['low'] if 'low' in 'infrastructure' else '', + ThreshInfrastructureMedium = config['infrastructure']['medium'] if 'medium' in 'infrastructure' else '', + ThreshInfrastructureHigh = config['infrastructure']['high'] if 'high' in 'infrastructure' else '', + ThreshInfrastructureCritical = config['infrastructure']['critical'] if 'critical' in 'infrastructure' else '', + ThreshSastLow = config['sast']['low'] if 'low' in 'sast' else '', + ThreshSastMedium = config['sast']['medium'] if 'medium' in 'sast' else '', + ThreshSastHigh = config['sast']['high'] if 'high' in 'sast' else '', + ThreshSastCritical = config['sast']['critical'] if 'critical' in 'sast' else '', + ThreshIacLow = config['iac']['low'] if 'low' in 'iac' else '', + ThreshIacMedium = config['iac']['medium'] if 'medium' in 'iac' else '', + ThreshIacHigh = config['iac']['high'] if 'high' in 'iac' else '', + ThreshIacCritical = config['iac']['critical'] if 'critical' in 'iac' else '', + ThreshSecretsLow = config['secret']['low'] if 'low' in 'secret' else '', + ThreshSecretsMedium = config['secret']['medium'] if 'medium' in 'secret' else '', + ThreshSecretsHigh = config['secret']['high'] if 'high' in 'secret' else '', + ThreshSecretsCritical = config['secret']['critical'] if 'critical' in 'secret' else '', ) db.session.add(job) db_connection_handler(db) From 0cab74cda8c3939b81e5bbbdf39f2c35008df75b Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sun, 17 Mar 2024 18:22:04 -0700 Subject: [PATCH 40/51] Feature/update settings function (#501) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups --- src/vr/admin/models.py | 37 ++++ src/vr/admin/routes/settings.py | 303 +++++++++++++++++++++++---- src/vr/db_models/setup.py | 37 ++++ src/vr/templates/admin/settings.html | 281 ++++++++++++++++++++++++- 4 files changed, 615 insertions(+), 43 deletions(-) diff --git a/src/vr/admin/models.py b/src/vr/admin/models.py index 1048b3d0..89785772 100644 --- a/src/vr/admin/models.py +++ b/src/vr/admin/models.py @@ -416,6 +416,43 @@ class AppConfig(db.Model): __tablename__ = 'AppConfig' id = db.Column(db.Integer, primary_key=True) first_access = db.Column(db.Boolean, nullable=False, default=True) + settings_initialized = db.Column(db.Boolean, nullable=False, default=False) + APP_EXT_URL = db.Column(db.String(200)) + AUTH_TYPE = db.Column(db.String(200)) + AZAD_AUTHORITY = db.Column(db.String(200)) + AZAD_CLIENT_ID = db.Column(db.String(200)) + AZAD_CLIENT_SECRET = db.Column(db.String(200)) + AZURE_KEYVAULT_NAME = db.Column(db.String(200)) + ENV = db.Column(db.String(200)) + INSECURE_OAUTH = db.Column(db.String(200)) + JENKINS_HOST = db.Column(db.String(200)) + JENKINS_KEY = db.Column(db.String(200)) + JENKINS_PROJECT = db.Column(db.String(200)) + JENKINS_STAGING_PROJECT = db.Column(db.String(200)) + JENKINS_TOKEN = db.Column(db.String(200)) + JENKINS_USER = db.Column(db.String(200)) + LDAP_BASE_DN = db.Column(db.String(200)) + LDAP_BIND_USER_DN = db.Column(db.String(200)) + LDAP_BIND_USER_PASSWORD = db.Column(db.String(200)) + LDAP_GROUP_DN = db.Column(db.String(200)) + LDAP_HOST = db.Column(db.String(200)) + LDAP_PORT = db.Column(db.String(200)) + LDAP_USER_DN = db.Column(db.String(200)) + LDAP_USER_LOGIN_ATTR = db.Column(db.String(200)) + LDAP_USER_RDN_ATTR = db.Column(db.String(200)) + PROD_DB_URI = db.Column(db.String(200)) + SMTP_ADMIN_EMAIL = db.Column(db.String(200)) + SMTP_HOST = db.Column(db.String(200)) + SMTP_PASSWORD = db.Column(db.String(200)) + SMTP_USER = db.Column(db.String(200)) + SNOW_CLIENT_ID = db.Column(db.String(200)) + SNOW_CLIENT_SECRET = db.Column(db.String(200)) + SNOW_INSTANCE_NAME = db.Column(db.String(200)) + SNOW_PASSWORD = db.Column(db.String(200)) + SNOW_USERNAME = db.Column(db.String(200)) + VERSION = db.Column(db.String(200)) + JENKINS_ENABLED = db.Column(db.String(200)) + SNOW_ENABLED = db.Column(db.String(200)) class SuSiteConfiguration(db.Model): diff --git a/src/vr/admin/routes/settings.py b/src/vr/admin/routes/settings.py index c366ce43..a3125e54 100644 --- a/src/vr/admin/routes/settings.py +++ b/src/vr/admin/routes/settings.py @@ -1,4 +1,4 @@ -from flask import session, redirect, url_for, render_template +from flask import session, redirect, url_for, render_template, request from flask_login import login_required from vr import db, app import os @@ -10,14 +10,18 @@ AZAD_CLIENT_ID, AZAD_CLIENT_SECRET, AZAD_AUTHORITY, JENKINS_USER, AZURE_KEYVAULT_NAME, INSECURE_OAUTH, \ JENKINS_HOST, JENKINS_KEY, JENKINS_PROJECT, JENKINS_STAGING_PROJECT, JENKINS_TOKEN, SMTP_ADMIN_EMAIL, \ SMTP_HOST, SMTP_PASSWORD, SMTP_USER, SNOW_CLIENT_ID, SNOW_CLIENT_SECRET, SNOW_INSTANCE_NAME, SNOW_PASSWORD, \ - SNOW_USERNAME, VERSION + SNOW_USERNAME, VERSION, JENKINS_ENABLED, SNOW_ENABLED from flask_sqlalchemy import SQLAlchemy +from vr.admin.models import AppConfig +from vr.admin.functions import db_connection_handler +from sqlalchemy import text + NAV = { 'CAT': { "name": "Settings", "url": "admin.admin_dashboard"} } -@admin.route('/settings', methods=['GET']) +@admin.route('/settings', methods=['GET', 'POST']) @login_required def settings(): NAV['curpage'] = {"name": "Settings"} @@ -27,44 +31,263 @@ def settings(): elif status == 403: return render_template('403.html', user=user, nav_cat={}, nav_subcat='', \ nav_subsubcat='', nav_curpage={"name": "Unauthorized"}) - current_settings = { - "APP_EXT_URL": APP_EXT_URL, - "AUTH_TYPE": AUTH_TYPE, - "AZAD_AUTHORITY":AZAD_AUTHORITY, - "AZAD_CLIENT_ID": AZAD_CLIENT_ID, - "AZAD_CLIENT_SECRET": AZAD_CLIENT_SECRET, - "AZURE_KEYVAULT_NAME": AZURE_KEYVAULT_NAME, - "ENV": ENV, - "INSECURE_OAUTH": INSECURE_OAUTH, - "JENKINS_HOST": JENKINS_HOST, - "JENKINS_KEY": JENKINS_KEY, - "JENKINS_PROJECT": JENKINS_PROJECT, - "JENKINS_STAGING_PROJECT": JENKINS_STAGING_PROJECT, - "JENKINS_TOKEN": JENKINS_TOKEN, - "JENKINS_USER": JENKINS_USER, - "LDAP_BASE_DN": LDAP_BASE_DN, - "LDAP_BIND_USER_DN": LDAP_BIND_USER_DN, - "LDAP_BIND_USER_PASSWORD": LDAP_BIND_USER_PASSWORD, - "LDAP_GROUP_DN": LDAP_GROUP_DN, - "LDAP_HOST": LDAP_HOST, - "LDAP_PORT": LDAP_PORT, - "LDAP_USER_DN": LDAP_USER_DN, - "LDAP_USER_LOGIN_ATTR": LDAP_USER_LOGIN_ATTR, - "LDAP_USER_RDN_ATTR": LDAP_USER_RDN_ATTR, - "PROD_DB_URI": PROD_DB_URI, - "SMTP_ADMIN_EMAIL": SMTP_ADMIN_EMAIL, - "SMTP_HOST": SMTP_HOST, - "SMTP_PASSWORD": SMTP_PASSWORD, - "SMTP_USER": SMTP_USER, - "SNOW_CLIENT_ID": SNOW_CLIENT_ID, - "SNOW_CLIENT_SECRET": SNOW_CLIENT_SECRET, - "SNOW_INSTANCE_NAME": SNOW_INSTANCE_NAME, - "SNOW_PASSWORD": SNOW_PASSWORD, - "SNOW_USERNAME": SNOW_USERNAME, - "VERSION": VERSION, - } + if request.method == 'POST': + app_config = AppConfig.query.first() + + all = request.form + update_json = { + AppConfig.JENKINS_ENABLED: all["JENKINS_ENABLED"], + AppConfig.SNOW_ENABLED: all["SNOW_ENABLED"], + AppConfig.APP_EXT_URL: all["APP_EXT_URL"], + AppConfig.AUTH_TYPE: all["AUTH_TYPE"], + AppConfig.AZAD_AUTHORITY: all["AZAD_AUTHORITY"], + AppConfig.AZAD_CLIENT_ID: all["AZAD_CLIENT_ID"], + AppConfig.AZAD_CLIENT_SECRET: all["AZAD_CLIENT_SECRET"], + AppConfig.AZURE_KEYVAULT_NAME: all["AZURE_KEYVAULT_NAME"], + AppConfig.ENV: all["ENV"], + AppConfig.INSECURE_OAUTH: all["INSECURE_OAUTH"], + AppConfig.JENKINS_HOST: all["JENKINS_HOST"], + AppConfig.JENKINS_KEY: all["JENKINS_KEY"], + AppConfig.JENKINS_PROJECT: all["JENKINS_PROJECT"], + AppConfig.JENKINS_STAGING_PROJECT: all["JENKINS_STAGING_PROJECT"], + AppConfig.JENKINS_TOKEN: all["JENKINS_TOKEN"], + AppConfig.JENKINS_USER: all["JENKINS_USER"], + AppConfig.LDAP_BASE_DN: all["LDAP_BASE_DN"], + AppConfig.LDAP_BIND_USER_DN: all["LDAP_BIND_USER_DN"], + AppConfig.LDAP_BIND_USER_PASSWORD: all["LDAP_BIND_USER_PASSWORD"], + AppConfig.LDAP_GROUP_DN: all["LDAP_GROUP_DN"], + AppConfig.LDAP_HOST: all["LDAP_HOST"], + AppConfig.LDAP_PORT: all["LDAP_PORT"], + AppConfig.LDAP_USER_DN: all["LDAP_USER_DN"], + AppConfig.LDAP_USER_LOGIN_ATTR: all["LDAP_USER_LOGIN_ATTR"], + AppConfig.LDAP_USER_RDN_ATTR: all["LDAP_USER_RDN_ATTR"], + AppConfig.PROD_DB_URI: all["PROD_DB_URI"], + AppConfig.SMTP_ADMIN_EMAIL: all["SMTP_ADMIN_EMAIL"], + AppConfig.SMTP_HOST: all["SMTP_HOST"], + AppConfig.SMTP_PASSWORD: all["SMTP_PASSWORD"], + AppConfig.SMTP_USER: all["SMTP_USER"], + AppConfig.SNOW_CLIENT_ID: all["SNOW_CLIENT_ID"], + AppConfig.SNOW_CLIENT_SECRET: all["SNOW_CLIENT_SECRET"], + AppConfig.SNOW_INSTANCE_NAME: all["SNOW_INSTANCE_NAME"], + AppConfig.SNOW_PASSWORD: all["SNOW_PASSWORD"], + AppConfig.SNOW_USERNAME: all["SNOW_USERNAME"], + AppConfig.VERSION: all["VERSION"], + } + if not app_config.settings_initialized: + update_json[AppConfig.settings_initialized] = True + db.session.query(AppConfig) \ + .update(update_json, synchronize_session=False) + db_connection_handler(db) + set_env_variables(all) + current_settings = { + "JENKINS_ENABLED": all["JENKINS_ENABLED"], + "SNOW_ENABLED": all["SNOW_ENABLED"], + "APP_EXT_URL": all["APP_EXT_URL"], + "AUTH_TYPE": all["AUTH_TYPE"], + "AZAD_AUTHORITY": all["AZAD_AUTHORITY"], + "AZAD_CLIENT_ID": all["AZAD_CLIENT_ID"], + "AZAD_CLIENT_SECRET": all["AZAD_CLIENT_SECRET"], + "AZURE_KEYVAULT_NAME": all["AZURE_KEYVAULT_NAME"], + "ENV": all["ENV"], + "INSECURE_OAUTH": all["INSECURE_OAUTH"], + "JENKINS_HOST": all["JENKINS_HOST"], + "JENKINS_KEY": all["JENKINS_KEY"], + "JENKINS_PROJECT": all["JENKINS_PROJECT"], + "JENKINS_STAGING_PROJECT": all["JENKINS_STAGING_PROJECT"], + "JENKINS_TOKEN": all["JENKINS_TOKEN"], + "JENKINS_USER": all["JENKINS_USER"], + "LDAP_BASE_DN": all["LDAP_BASE_DN"], + "LDAP_BIND_USER_DN": all["LDAP_BIND_USER_DN"], + "LDAP_BIND_USER_PASSWORD": all["LDAP_BIND_USER_PASSWORD"], + "LDAP_GROUP_DN": all["LDAP_GROUP_DN"], + "LDAP_HOST": all["LDAP_HOST"], + "LDAP_PORT": all["LDAP_PORT"], + "LDAP_USER_DN": all["LDAP_USER_DN"], + "LDAP_USER_LOGIN_ATTR": all["LDAP_USER_LOGIN_ATTR"], + "LDAP_USER_RDN_ATTR": all["LDAP_USER_RDN_ATTR"], + "PROD_DB_URI": all["PROD_DB_URI"], + "SMTP_ADMIN_EMAIL": all["SMTP_ADMIN_EMAIL"], + "SMTP_HOST": all["SMTP_HOST"], + "SMTP_PASSWORD": all["SMTP_PASSWORD"], + "SMTP_USER": all["SMTP_USER"], + "SNOW_CLIENT_ID": all["SNOW_CLIENT_ID"], + "SNOW_CLIENT_SECRET": all["SNOW_CLIENT_SECRET"], + "SNOW_INSTANCE_NAME": all["SNOW_INSTANCE_NAME"], + "SNOW_PASSWORD": all["SNOW_PASSWORD"], + "SNOW_USERNAME": all["SNOW_USERNAME"], + "VERSION": all["VERSION"], + } + else: + app_config = AppConfig.query.first() + if app_config.settings_initialized: + current_settings = { + "JENKINS_ENABLED": app_config.JENKINS_ENABLED, + "SNOW_ENABLED": app_config.SNOW_ENABLED, + "APP_EXT_URL": app_config.APP_EXT_URL, + "AUTH_TYPE": app_config.AUTH_TYPE, + "AZAD_AUTHORITY": app_config.AZAD_AUTHORITY, + "AZAD_CLIENT_ID": app_config.AZAD_CLIENT_ID, + "AZAD_CLIENT_SECRET": app_config.AZAD_CLIENT_SECRET, + "AZURE_KEYVAULT_NAME": app_config.AZURE_KEYVAULT_NAME, + "ENV": app_config.ENV, + "INSECURE_OAUTH": app_config.INSECURE_OAUTH, + "JENKINS_HOST": app_config.JENKINS_HOST, + "JENKINS_KEY": app_config.JENKINS_KEY, + "JENKINS_PROJECT": app_config.JENKINS_PROJECT, + "JENKINS_STAGING_PROJECT": app_config.JENKINS_STAGING_PROJECT, + "JENKINS_USER": app_config.JENKINS_USER, + "JENKINS_TOKEN": app_config.JENKINS_TOKEN, + "LDAP_BASE_DN": app_config.LDAP_BASE_DN, + "LDAP_BIND_USER_DN": app_config.LDAP_BIND_USER_DN, + "LDAP_BIND_USER_PASSWORD": app_config.LDAP_BIND_USER_PASSWORD, + "LDAP_GROUP_DN": app_config.LDAP_GROUP_DN, + "LDAP_HOST": app_config.LDAP_HOST, + "LDAP_PORT": app_config.LDAP_PORT, + "LDAP_USER_DN": app_config.LDAP_USER_DN, + "LDAP_USER_LOGIN_ATTR": app_config.LDAP_USER_LOGIN_ATTR, + "LDAP_USER_RDN_ATTR": app_config.LDAP_USER_RDN_ATTR, + "PROD_DB_URI": app_config.PROD_DB_URI, + "SMTP_ADMIN_EMAIL": app_config.SMTP_ADMIN_EMAIL, + "SMTP_HOST": app_config.SMTP_HOST, + "SMTP_USER": app_config.SMTP_USER, + "SMTP_PASSWORD": app_config.SMTP_PASSWORD, + "SNOW_CLIENT_ID": app_config.SNOW_CLIENT_ID, + "SNOW_CLIENT_SECRET": app_config.SNOW_CLIENT_SECRET, + "SNOW_INSTANCE_NAME": app_config.SNOW_INSTANCE_NAME, + "SNOW_USERNAME": app_config.SNOW_USERNAME, + "SNOW_PASSWORD": app_config.SNOW_PASSWORD, + "VERSION": app_config.VERSION, + } + else: + current_settings = { + "JENKINS_ENABLED": JENKINS_ENABLED, + "SNOW_ENABLED": SNOW_ENABLED, + "APP_EXT_URL": APP_EXT_URL, + "AUTH_TYPE": AUTH_TYPE, + "AZAD_AUTHORITY":AZAD_AUTHORITY, + "AZAD_CLIENT_ID": AZAD_CLIENT_ID, + "AZAD_CLIENT_SECRET": AZAD_CLIENT_SECRET, + "AZURE_KEYVAULT_NAME": AZURE_KEYVAULT_NAME, + "ENV": ENV, + "INSECURE_OAUTH": INSECURE_OAUTH, + "JENKINS_HOST": JENKINS_HOST, + "JENKINS_KEY": JENKINS_KEY, + "JENKINS_PROJECT": JENKINS_PROJECT, + "JENKINS_STAGING_PROJECT": JENKINS_STAGING_PROJECT, + "JENKINS_USER": JENKINS_USER, + "JENKINS_TOKEN": JENKINS_TOKEN, + "LDAP_BASE_DN": LDAP_BASE_DN, + "LDAP_BIND_USER_DN": LDAP_BIND_USER_DN, + "LDAP_BIND_USER_PASSWORD": LDAP_BIND_USER_PASSWORD, + "LDAP_GROUP_DN": LDAP_GROUP_DN, + "LDAP_HOST": LDAP_HOST, + "LDAP_PORT": LDAP_PORT, + "LDAP_USER_DN": LDAP_USER_DN, + "LDAP_USER_LOGIN_ATTR": LDAP_USER_LOGIN_ATTR, + "LDAP_USER_RDN_ATTR": LDAP_USER_RDN_ATTR, + "PROD_DB_URI": PROD_DB_URI, + "SMTP_ADMIN_EMAIL": SMTP_ADMIN_EMAIL, + "SMTP_HOST": SMTP_HOST, + "SMTP_USER": SMTP_USER, + "SMTP_PASSWORD": SMTP_PASSWORD, + "SNOW_CLIENT_ID": SNOW_CLIENT_ID, + "SNOW_CLIENT_SECRET": SNOW_CLIENT_SECRET, + "SNOW_INSTANCE_NAME": SNOW_INSTANCE_NAME, + "SNOW_USERNAME": SNOW_USERNAME, + "SNOW_PASSWORD": SNOW_PASSWORD, + "VERSION": VERSION, + } + cat_general = [ + 'APP_EXT_URL', + 'AUTH_TYPE', + 'ENV', + 'INSECURE_OAUTH', + 'PROD_DB_URI', + 'VERSION', + 'AZURE_KEYVAULT_NAME' + ] + cat_azad = [ + 'AZAD_AUTHORITY', + 'AZAD_CLIENT_ID', + 'AZAD_CLIENT_SECRET' + ] + cat_jenkins = [ + 'JENKINS_ENABLED', + 'JENKINS_HOST', + 'JENKINS_KEY', + 'JENKINS_PROJECT', + 'JENKINS_STAGING_PROJECT', + 'JENKINS_TOKEN', + 'JENKINS_USER' + ] + cat_ldap = [ + 'LDAP_BASE_DN', + 'LDAP_BIND_USER_DN', + 'LDAP_BIND_USER_PASSWORD', + 'LDAP_GROUP_DN', + 'LDAP_HOST', + 'LDAP_PORT', + 'LDAP_USER_DN', + 'LDAP_USER_LOGIN_ATTR', + 'LDAP_USER_RDN_ATTR' + ] + smtp_settings = [ + 'SMTP_ADMIN_EMAIL', + 'SMTP_HOST', + 'SMTP_PASSWORD', + 'SMTP_USER' + ] + snow_settings = [ + 'SNOW_ENABLED', + 'SNOW_CLIENT_ID', + 'SNOW_CLIENT_SECRET', + 'SNOW_INSTANCE_NAME', + 'SNOW_PASSWORD', + 'SNOW_USERNAME' + ] return render_template('admin/settings.html', user_roles=user_roles, NAV=NAV, - user=user, settings=current_settings) + user=user, settings=current_settings, cat_general=cat_general, + cat_azad=cat_azad, cat_jenkins=cat_jenkins, cat_ldap=cat_ldap, + smtp_settings=smtp_settings, snow_settings=snow_settings) + +def set_env_variables(form): + os.environ['APP_EXT_URL'] = form["APP_EXT_URL"] + os.environ['AUTH_TYPE'] = form["AUTH_TYPE"] + os.environ['AZAD_AUTHORITY'] = form["AZAD_AUTHORITY"] + os.environ['AZAD_CLIENT_ID'] = form["AZAD_CLIENT_ID"] + os.environ['AZAD_CLIENT_SECRET'] = form["AZAD_CLIENT_SECRET"] + os.environ['AZURE_KEYVAULT_NAME'] = form["AZURE_KEYVAULT_NAME"] + os.environ['ENV'] = form["ENV"] + os.environ['INSECURE_OAUTH'] = form["INSECURE_OAUTH"] + os.environ['JENKINS_ENABLED'] = form["JENKINS_ENABLED"] + os.environ['JENKINS_HOST'] = form["JENKINS_HOST"] + os.environ['JENKINS_KEY'] = form["JENKINS_KEY"] + os.environ['JENKINS_PROJECT'] = form["JENKINS_PROJECT"] + os.environ['JENKINS_STAGING_PROJECT'] = form["JENKINS_STAGING_PROJECT"] + os.environ['JENKINS_TOKEN'] = form["JENKINS_TOKEN"] + os.environ['JENKINS_USER'] = form["JENKINS_USER"] + os.environ['LDAP_BASE_DN'] = form["LDAP_BASE_DN"] + os.environ['LDAP_BIND_USER_DN'] = form["LDAP_BIND_USER_DN"] + os.environ['LDAP_BIND_USER_PASSWORD'] = form["LDAP_BIND_USER_PASSWORD"] + os.environ['LDAP_GROUP_DN'] = form["LDAP_GROUP_DN"] + os.environ['LDAP_HOST'] = form["LDAP_HOST"] + os.environ['LDAP_PORT'] = form["LDAP_PORT"] + os.environ['LDAP_USER_DN'] = form["LDAP_USER_DN"] + os.environ['LDAP_USER_LOGIN_ATTR'] = form["LDAP_USER_LOGIN_ATTR"] + os.environ['LDAP_USER_RDN_ATTR'] = form["LDAP_USER_RDN_ATTR"] + os.environ['PROD_DB_URI'] = form["PROD_DB_URI"] + os.environ['SMTP_ADMIN_EMAIL'] = form["SMTP_ADMIN_EMAIL"] + os.environ['SMTP_HOST'] = form["SMTP_HOST"] + os.environ['SMTP_PASSWORD'] = form["SMTP_PASSWORD"] + os.environ['SMTP_USER'] = form["SMTP_USER"] + os.environ['SNOW_ENABLED'] = form["SNOW_ENABLED"] + os.environ['SNOW_CLIENT_ID'] = form["SNOW_CLIENT_ID"] + os.environ['SNOW_CLIENT_SECRET'] = form["SNOW_CLIENT_SECRET"] + os.environ['SNOW_INSTANCE_NAME'] = form["SNOW_INSTANCE_NAME"] + os.environ['SNOW_PASSWORD'] = form["SNOW_PASSWORD"] + os.environ['SNOW_USERNAME'] = form["SNOW_USERNAME"] + os.environ['VERSION'] = form["VERSION"] + @admin.route('/dangerous/delete_all', methods=['POST']) def delete_all_data(): diff --git a/src/vr/db_models/setup.py b/src/vr/db_models/setup.py index 67adfc3b..b36fdc22 100644 --- a/src/vr/db_models/setup.py +++ b/src/vr/db_models/setup.py @@ -117,6 +117,43 @@ class AppConfig(db.Model): __table_args__ = {'extend_existing': True} id = db.Column(db.Integer, primary_key=True) first_access = db.Column(db.Boolean, nullable=False, default=True) + settings_initialized = db.Column(db.Boolean, nullable=False, default=False) + APP_EXT_URL = db.Column(db.String(200)) + AUTH_TYPE = db.Column(db.String(200)) + AZAD_AUTHORITY = db.Column(db.String(200)) + AZAD_CLIENT_ID = db.Column(db.String(200)) + AZAD_CLIENT_SECRET = db.Column(db.String(200)) + AZURE_KEYVAULT_NAME = db.Column(db.String(200)) + ENV = db.Column(db.String(200)) + INSECURE_OAUTH = db.Column(db.String(200)) + JENKINS_HOST = db.Column(db.String(200)) + JENKINS_KEY = db.Column(db.String(200)) + JENKINS_PROJECT = db.Column(db.String(200)) + JENKINS_STAGING_PROJECT = db.Column(db.String(200)) + JENKINS_TOKEN = db.Column(db.String(200)) + JENKINS_USER = db.Column(db.String(200)) + LDAP_BASE_DN = db.Column(db.String(200)) + LDAP_BIND_USER_DN = db.Column(db.String(200)) + LDAP_BIND_USER_PASSWORD = db.Column(db.String(200)) + LDAP_GROUP_DN = db.Column(db.String(200)) + LDAP_HOST = db.Column(db.String(200)) + LDAP_PORT = db.Column(db.String(200)) + LDAP_USER_DN = db.Column(db.String(200)) + LDAP_USER_LOGIN_ATTR = db.Column(db.String(200)) + LDAP_USER_RDN_ATTR = db.Column(db.String(200)) + PROD_DB_URI = db.Column(db.String(200)) + SMTP_ADMIN_EMAIL = db.Column(db.String(200)) + SMTP_HOST = db.Column(db.String(200)) + SMTP_PASSWORD = db.Column(db.String(200)) + SMTP_USER = db.Column(db.String(200)) + SNOW_CLIENT_ID = db.Column(db.String(200)) + SNOW_CLIENT_SECRET = db.Column(db.String(200)) + SNOW_INSTANCE_NAME = db.Column(db.String(200)) + SNOW_PASSWORD = db.Column(db.String(200)) + SNOW_USERNAME = db.Column(db.String(200)) + VERSION = db.Column(db.String(200)) + JENKINS_ENABLED = db.Column(db.String(200)) + SNOW_ENABLED = db.Column(db.String(200)) AppConfig() diff --git a/src/vr/templates/admin/settings.html b/src/vr/templates/admin/settings.html index 06a4bee5..a7564e1a 100644 --- a/src/vr/templates/admin/settings.html +++ b/src/vr/templates/admin/settings.html @@ -113,13 +113,135 @@

Application Settings

+ + +
+

General Settings

+ {% for key, value in settings.items() %} + {% if key in cat_general %} +
+ + {% if key == "AUTH_TYPE" %} + + {% elif key == "INSECURE_OAUTH" %} + + {% elif key == "PROD_DB_URI" %} +
+ {% elif key == "ENV" or key == "VERSION" %} + + {% else %} + + {% endif %} +
+ {% endif %} + {% endfor %} +
+ +
+

LDAP Settings

{% for key, value in settings.items() %} -
- - + {% if key in cat_ldap %} +
+ + +
+ {% endif %} + {% endfor %}
+ +
+

Azure Active Directory Settings

+ {% for key, value in settings.items() %} + {% if key in cat_azad %} +
+ + +
+ {% endif %} {% endfor %} +
+ +
+

SMTP Settings

+ {% for key, value in settings.items() %} + {% if key in smtp_settings %} +
+ + {% if key == 'SMTP_PASSWORD' %} +
+ {% else %} + + {% endif %} + +
+ {% endif %} + {% endfor %} +
+ +
+

Jenkins Settings

+ {% for key, value in settings.items() %} + {% if key in cat_jenkins %} +
+ {% if key == 'JENKINS_ENABLED' %} + + + {% elif key == 'JENKINS_KEY' %} + +
+ {% elif key == 'JENKINS_TOKEN' %} + +
+ {% else %} + + + {% endif %} +
+ {% endif %} + {% endfor %} +
+ + + + + +
+

ServiceNOW Settings

+ {% for key, value in settings.items() %} + {% if key in snow_settings %} +
+ {% if key == 'SNOW_ENABLED' %} + + + {% elif key == 'SNOW_CLIENT_SECRET' %} + +
+ {% elif key == 'SNOW_PASSWORD' %} + +
+ {% else %} + + + {% endif %} +
+ {% endif %} + {% endfor %} +
+
@@ -128,4 +250,157 @@

Application Settings

+ + {% endblock %} From a59813684be8284c77d3121f0870c5e45dbe3ad0 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sun, 17 Mar 2024 21:51:13 -0700 Subject: [PATCH 41/51] Feature/update release based db settings (#503) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates --- src/vr/__init__.py | 5 ++++ src/vr/db_models/updates.py | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 src/vr/db_models/updates.py diff --git a/src/vr/__init__.py b/src/vr/__init__.py index e1bc03cf..a35d9183 100644 --- a/src/vr/__init__.py +++ b/src/vr/__init__.py @@ -29,6 +29,7 @@ from Crypto.PublicKey import RSA from Crypto.Cipher import PKCS1_OAEP from requests.auth import HTTPBasicAuth +from vr.db_models.updates import createNewTables if AUTH_TYPE == 'azuread': from flask_session import Session @@ -180,6 +181,10 @@ def base64encode(value): return None +## Release-based updates ## +cwd = os.getcwd() +createNewTables(DB_URI) +print() ## Cronjob-like tasks section ## def train_model_every_six_hours(): scheduler = BackgroundScheduler() diff --git a/src/vr/db_models/updates.py b/src/vr/db_models/updates.py new file mode 100644 index 00000000..d6028a0f --- /dev/null +++ b/src/vr/db_models/updates.py @@ -0,0 +1,56 @@ +from flask_sqlalchemy import SQLAlchemy +from flask import Flask + + +def createNewTables(db_uri): + mock_app = Flask(__name__) + # Example database URI, replace it with your actual database URI + mock_app.config['SQLALCHEMY_DATABASE_URI'] = db_uri + mock_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + db = SQLAlchemy(mock_app) + + class AppConfig(db.Model): + __tablename__ = 'AppConfig' + __table_args__ = {'extend_existing': True} + id = db.Column(db.Integer, primary_key=True) + first_access = db.Column(db.Boolean, nullable=False, default=True) + settings_initialized = db.Column(db.Boolean, nullable=False, default=False) + APP_EXT_URL = db.Column(db.String(200)) + AUTH_TYPE = db.Column(db.String(200)) + AZAD_AUTHORITY = db.Column(db.String(200)) + AZAD_CLIENT_ID = db.Column(db.String(200)) + AZAD_CLIENT_SECRET = db.Column(db.String(200)) + AZURE_KEYVAULT_NAME = db.Column(db.String(200)) + ENV = db.Column(db.String(200)) + INSECURE_OAUTH = db.Column(db.String(200)) + JENKINS_HOST = db.Column(db.String(200)) + JENKINS_KEY = db.Column(db.String(200)) + JENKINS_PROJECT = db.Column(db.String(200)) + JENKINS_STAGING_PROJECT = db.Column(db.String(200)) + JENKINS_TOKEN = db.Column(db.String(200)) + JENKINS_USER = db.Column(db.String(200)) + LDAP_BASE_DN = db.Column(db.String(200)) + LDAP_BIND_USER_DN = db.Column(db.String(200)) + LDAP_BIND_USER_PASSWORD = db.Column(db.String(200)) + LDAP_GROUP_DN = db.Column(db.String(200)) + LDAP_HOST = db.Column(db.String(200)) + LDAP_PORT = db.Column(db.String(200)) + LDAP_USER_DN = db.Column(db.String(200)) + LDAP_USER_LOGIN_ATTR = db.Column(db.String(200)) + LDAP_USER_RDN_ATTR = db.Column(db.String(200)) + PROD_DB_URI = db.Column(db.String(200)) + SMTP_ADMIN_EMAIL = db.Column(db.String(200)) + SMTP_HOST = db.Column(db.String(200)) + SMTP_PASSWORD = db.Column(db.String(200)) + SMTP_USER = db.Column(db.String(200)) + SNOW_CLIENT_ID = db.Column(db.String(200)) + SNOW_CLIENT_SECRET = db.Column(db.String(200)) + SNOW_INSTANCE_NAME = db.Column(db.String(200)) + SNOW_PASSWORD = db.Column(db.String(200)) + SNOW_USERNAME = db.Column(db.String(200)) + VERSION = db.Column(db.String(200)) + JENKINS_ENABLED = db.Column(db.String(200)) + SNOW_ENABLED = db.Column(db.String(200)) + + with mock_app.app_context(): + db.create_all() From bc17dcc362fff6a672633332c6c0e28fbaf8f2fa Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sun, 17 Mar 2024 23:08:12 -0700 Subject: [PATCH 42/51] Feature/update release based db settings (#505) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates --- src/vr/__init__.py | 5 +- src/vr/db_models/updates.py | 140 ++++++++++++++++++++++-------------- 2 files changed, 90 insertions(+), 55 deletions(-) diff --git a/src/vr/__init__.py b/src/vr/__init__.py index a35d9183..42a86ea1 100644 --- a/src/vr/__init__.py +++ b/src/vr/__init__.py @@ -182,9 +182,8 @@ def base64encode(value): ## Release-based updates ## -cwd = os.getcwd() -createNewTables(DB_URI) -print() +createNewTables(app) + ## Cronjob-like tasks section ## def train_model_every_six_hours(): scheduler = BackgroundScheduler() diff --git a/src/vr/db_models/updates.py b/src/vr/db_models/updates.py index d6028a0f..93106286 100644 --- a/src/vr/db_models/updates.py +++ b/src/vr/db_models/updates.py @@ -1,56 +1,92 @@ -from flask_sqlalchemy import SQLAlchemy -from flask import Flask +import mysql.connector +import sqlite3 +import os -def createNewTables(db_uri): - mock_app = Flask(__name__) - # Example database URI, replace it with your actual database URI - mock_app.config['SQLALCHEMY_DATABASE_URI'] = db_uri - mock_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - db = SQLAlchemy(mock_app) +def get_client(app): + if app.config['RUNTIME_ENV'] == 'test': + cur_path = os.getcwd() + if 'www' in cur_path and 'html' in cur_path: + db_uri = '/var/www/html/src/instance/database.db' + else: + db_uri = 'instance/database.db' + db = sqlite3.connect(db_uri) + cur = db.cursor() + return cur, db + else: + db_uri = app.config['SQLALCHEMY_DATABASE_URI'] + main_part = db_uri.split('://')[1] + un = main_part.split(':', 1)[0] + db_name = main_part.rsplit('/', 1)[1] + host_and_port = main_part.rsplit('@', 1)[1].replace(f"/{db_name}", '') + host = host_and_port.split(':')[0] + port = int(host_and_port.split(':')[1]) + pw = main_part.split(':', 1)[1].replace(f"@{host}", '').replace(f"/{db_name}", '').replace(f":{port}", "") + db = mysql.connector.connect(host=host, database=db_name, user=un, password=pw, port=port) + cur = db.cursor() + return cur, db - class AppConfig(db.Model): - __tablename__ = 'AppConfig' - __table_args__ = {'extend_existing': True} - id = db.Column(db.Integer, primary_key=True) - first_access = db.Column(db.Boolean, nullable=False, default=True) - settings_initialized = db.Column(db.Boolean, nullable=False, default=False) - APP_EXT_URL = db.Column(db.String(200)) - AUTH_TYPE = db.Column(db.String(200)) - AZAD_AUTHORITY = db.Column(db.String(200)) - AZAD_CLIENT_ID = db.Column(db.String(200)) - AZAD_CLIENT_SECRET = db.Column(db.String(200)) - AZURE_KEYVAULT_NAME = db.Column(db.String(200)) - ENV = db.Column(db.String(200)) - INSECURE_OAUTH = db.Column(db.String(200)) - JENKINS_HOST = db.Column(db.String(200)) - JENKINS_KEY = db.Column(db.String(200)) - JENKINS_PROJECT = db.Column(db.String(200)) - JENKINS_STAGING_PROJECT = db.Column(db.String(200)) - JENKINS_TOKEN = db.Column(db.String(200)) - JENKINS_USER = db.Column(db.String(200)) - LDAP_BASE_DN = db.Column(db.String(200)) - LDAP_BIND_USER_DN = db.Column(db.String(200)) - LDAP_BIND_USER_PASSWORD = db.Column(db.String(200)) - LDAP_GROUP_DN = db.Column(db.String(200)) - LDAP_HOST = db.Column(db.String(200)) - LDAP_PORT = db.Column(db.String(200)) - LDAP_USER_DN = db.Column(db.String(200)) - LDAP_USER_LOGIN_ATTR = db.Column(db.String(200)) - LDAP_USER_RDN_ATTR = db.Column(db.String(200)) - PROD_DB_URI = db.Column(db.String(200)) - SMTP_ADMIN_EMAIL = db.Column(db.String(200)) - SMTP_HOST = db.Column(db.String(200)) - SMTP_PASSWORD = db.Column(db.String(200)) - SMTP_USER = db.Column(db.String(200)) - SNOW_CLIENT_ID = db.Column(db.String(200)) - SNOW_CLIENT_SECRET = db.Column(db.String(200)) - SNOW_INSTANCE_NAME = db.Column(db.String(200)) - SNOW_PASSWORD = db.Column(db.String(200)) - SNOW_USERNAME = db.Column(db.String(200)) - VERSION = db.Column(db.String(200)) - JENKINS_ENABLED = db.Column(db.String(200)) - SNOW_ENABLED = db.Column(db.String(200)) - with mock_app.app_context(): - db.create_all() +def createNewTables(app): + cur, db = get_client(app) + if app.config['RUNTIME_ENV'] == 'test': + sql = "PRAGMA table_info('AppConfig')" + else: + sql = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'vulnremediator' AND table_name = 'AppConfig'" + cur.execute(sql) + rows = cur.fetchall() + fields = [] + for i in rows: + fields.append(i[1]) + new_fields = [ + {"name": "APP_EXT_URL", "type": "VARCHAR", "char_num": 200}, + {"name": "AUTH_TYPE", "type": "VARCHAR", "char_num": 200}, + {"name": "AZAD_AUTHORITY", "type": "VARCHAR", "char_num": 200}, + {"name": "AZAD_CLIENT_ID", "type": "VARCHAR", "char_num": 200}, + {"name": "AZAD_CLIENT_SECRET", "type": "VARCHAR", "char_num": 200}, + {"name": "AZURE_KEYVAULT_NAME", "type": "VARCHAR", "char_num": 200}, + {"name": "ENV", "type": "VARCHAR", "char_num": 200}, + {"name": "INSECURE_OAUTH", "type": "VARCHAR", "char_num": 200}, + {"name": "JENKINS_HOST", "type": "VARCHAR", "char_num": 200}, + {"name": "JENKINS_KEY", "type": "VARCHAR", "char_num": 200}, + {"name": "JENKINS_PROJECT", "type": "VARCHAR", "char_num": 200}, + {"name": "JENKINS_STAGING_PROJECT", "type": "VARCHAR", "char_num": 200}, + {"name": "JENKINS_TOKEN", "type": "VARCHAR", "char_num": 200}, + {"name": "JENKINS_USER", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_BASE_DN", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_BIND_USER_DN", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_BIND_USER_PASSWORD", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_GROUP_DN", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_HOST", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_PORT", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_USER_DN", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_USER_LOGIN_ATTR", "type": "VARCHAR", "char_num": 200}, + {"name": "LDAP_USER_RDN_ATTR", "type": "VARCHAR", "char_num": 200}, + {"name": "PROD_DB_URI", "type": "VARCHAR", "char_num": 200}, + {"name": "SMTP_ADMIN_EMAIL", "type": "VARCHAR", "char_num": 200}, + {"name": "SMTP_HOST", "type": "VARCHAR", "char_num": 200}, + {"name": "SMTP_PASSWORD", "type": "VARCHAR", "char_num": 200}, + {"name": "SMTP_USER", "type": "VARCHAR", "char_num": 200}, + {"name": "SNOW_CLIENT_ID", "type": "VARCHAR", "char_num": 200}, + {"name": "SNOW_CLIENT_SECRET", "type": "VARCHAR", "char_num": 200}, + {"name": "SNOW_INSTANCE_NAME", "type": "VARCHAR", "char_num": 200}, + {"name": "SNOW_PASSWORD", "type": "VARCHAR", "char_num": 200}, + {"name": "SNOW_USERNAME", "type": "VARCHAR", "char_num": 200}, + {"name": "VERSION", "type": "VARCHAR", "char_num": 200}, + {"name": "JENKINS_ENABLED", "type": "VARCHAR", "char_num": 200}, + {"name": "SNOW_ENABLED", "type": "VARCHAR", "char_num": 200} + ] + + for i in new_fields: + if i['name'] not in fields: + if app.config['RUNTIME_ENV'] == 'test': + if i['type'] == 'VARCHAR': + var_stmt = f"VARCHAR({i['char_num']})" + sql = "ALTER TABLE AppConfig ADD COLUMN" + i['name'] + var_stmt + else: + if i['type'] == 'VARCHAR': + var_stmt = "TEXT" + sql = "ALTER TABLE AppConfig ADD COLUMN" + i['name'] + var_stmt + cur.execute(sql) + db.commit() + From 334bc6884965d92083006cf45eb5d35511681405 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Sun, 17 Mar 2024 23:20:31 -0700 Subject: [PATCH 43/51] Feature/fix db syntax (#507) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates From 24a95b38915db0fa3fac8ae8e91d878e2a1c0234 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 18 Mar 2024 06:56:45 -0700 Subject: [PATCH 44/51] Feature/fix db syntax (#509) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py --- src/vr/db_models/updates.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vr/db_models/updates.py b/src/vr/db_models/updates.py index 93106286..d9a2c8c8 100644 --- a/src/vr/db_models/updates.py +++ b/src/vr/db_models/updates.py @@ -37,7 +37,10 @@ def createNewTables(app): rows = cur.fetchall() fields = [] for i in rows: - fields.append(i[1]) + if app.config['RUNTIME_ENV'] == 'test': + fields.append(i[1]) + else: + fields.append(i[0]) new_fields = [ {"name": "APP_EXT_URL", "type": "VARCHAR", "char_num": 200}, {"name": "AUTH_TYPE", "type": "VARCHAR", "char_num": 200}, From 0b07ab020b78d5c769e4b0b067db886e8b126382 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 18 Mar 2024 07:13:57 -0700 Subject: [PATCH 45/51] Feature/fix syntax error (#511) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py * Update updates.py --- src/vr/db_models/updates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vr/db_models/updates.py b/src/vr/db_models/updates.py index d9a2c8c8..a3105305 100644 --- a/src/vr/db_models/updates.py +++ b/src/vr/db_models/updates.py @@ -85,11 +85,11 @@ def createNewTables(app): if app.config['RUNTIME_ENV'] == 'test': if i['type'] == 'VARCHAR': var_stmt = f"VARCHAR({i['char_num']})" - sql = "ALTER TABLE AppConfig ADD COLUMN" + i['name'] + var_stmt + sql = "ALTER TABLE AppConfig ADD COLUMN " + i['name'] + " " + var_stmt else: if i['type'] == 'VARCHAR': var_stmt = "TEXT" - sql = "ALTER TABLE AppConfig ADD COLUMN" + i['name'] + var_stmt + sql = "ALTER TABLE AppConfig ADD COLUMN " + i['name'] + " " + var_stmt cur.execute(sql) db.commit() From e246ba0561c49be9036c909ec6c79e39212aaf6c Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Mon, 18 Mar 2024 07:42:49 -0700 Subject: [PATCH 46/51] Feature/fix syntax error (#513) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py * Update updates.py * Update updates.py --- src/vr/db_models/updates.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/vr/db_models/updates.py b/src/vr/db_models/updates.py index a3105305..f1e28b1b 100644 --- a/src/vr/db_models/updates.py +++ b/src/vr/db_models/updates.py @@ -42,6 +42,7 @@ def createNewTables(app): else: fields.append(i[0]) new_fields = [ + {"name": "settings_initialized", "type": "BOOLEAN", "default": 0}, {"name": "APP_EXT_URL", "type": "VARCHAR", "char_num": 200}, {"name": "AUTH_TYPE", "type": "VARCHAR", "char_num": 200}, {"name": "AZAD_AUTHORITY", "type": "VARCHAR", "char_num": 200}, @@ -85,10 +86,14 @@ def createNewTables(app): if app.config['RUNTIME_ENV'] == 'test': if i['type'] == 'VARCHAR': var_stmt = f"VARCHAR({i['char_num']})" + elif i['type'] == 'BOOLEAN': + var_stmt = f"BOOLEAN DEFAULT {i['default']}" sql = "ALTER TABLE AppConfig ADD COLUMN " + i['name'] + " " + var_stmt else: if i['type'] == 'VARCHAR': var_stmt = "TEXT" + elif i['type'] == 'BOOLEAN': + var_stmt = f"BOOLEAN DEFAULT {i['default']}" sql = "ALTER TABLE AppConfig ADD COLUMN " + i['name'] + " " + var_stmt cur.execute(sql) db.commit() From f98f329338401f961c310188ae205e5b0aec2b1d Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Tue, 19 Mar 2024 19:24:16 -0700 Subject: [PATCH 47/51] Feature/fix syntax error (#515) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py * Update updates.py * Update updates.py * Update settings.py --- src/vr/admin/routes/settings.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/vr/admin/routes/settings.py b/src/vr/admin/routes/settings.py index a3125e54..fea3e3b5 100644 --- a/src/vr/admin/routes/settings.py +++ b/src/vr/admin/routes/settings.py @@ -44,7 +44,7 @@ def settings(): AppConfig.AZAD_CLIENT_ID: all["AZAD_CLIENT_ID"], AppConfig.AZAD_CLIENT_SECRET: all["AZAD_CLIENT_SECRET"], AppConfig.AZURE_KEYVAULT_NAME: all["AZURE_KEYVAULT_NAME"], - AppConfig.ENV: all["ENV"], + AppConfig.ENV: ENV, AppConfig.INSECURE_OAUTH: all["INSECURE_OAUTH"], AppConfig.JENKINS_HOST: all["JENKINS_HOST"], AppConfig.JENKINS_KEY: all["JENKINS_KEY"], @@ -71,7 +71,7 @@ def settings(): AppConfig.SNOW_INSTANCE_NAME: all["SNOW_INSTANCE_NAME"], AppConfig.SNOW_PASSWORD: all["SNOW_PASSWORD"], AppConfig.SNOW_USERNAME: all["SNOW_USERNAME"], - AppConfig.VERSION: all["VERSION"], + AppConfig.VERSION: VERSION, } if not app_config.settings_initialized: update_json[AppConfig.settings_initialized] = True @@ -88,7 +88,7 @@ def settings(): "AZAD_CLIENT_ID": all["AZAD_CLIENT_ID"], "AZAD_CLIENT_SECRET": all["AZAD_CLIENT_SECRET"], "AZURE_KEYVAULT_NAME": all["AZURE_KEYVAULT_NAME"], - "ENV": all["ENV"], + "ENV": ENV, "INSECURE_OAUTH": all["INSECURE_OAUTH"], "JENKINS_HOST": all["JENKINS_HOST"], "JENKINS_KEY": all["JENKINS_KEY"], @@ -115,7 +115,7 @@ def settings(): "SNOW_INSTANCE_NAME": all["SNOW_INSTANCE_NAME"], "SNOW_PASSWORD": all["SNOW_PASSWORD"], "SNOW_USERNAME": all["SNOW_USERNAME"], - "VERSION": all["VERSION"], + "VERSION": VERSION, } else: app_config = AppConfig.query.first() @@ -257,7 +257,7 @@ def set_env_variables(form): os.environ['AZAD_CLIENT_ID'] = form["AZAD_CLIENT_ID"] os.environ['AZAD_CLIENT_SECRET'] = form["AZAD_CLIENT_SECRET"] os.environ['AZURE_KEYVAULT_NAME'] = form["AZURE_KEYVAULT_NAME"] - os.environ['ENV'] = form["ENV"] + os.environ['ENV'] = ENV os.environ['INSECURE_OAUTH'] = form["INSECURE_OAUTH"] os.environ['JENKINS_ENABLED'] = form["JENKINS_ENABLED"] os.environ['JENKINS_HOST'] = form["JENKINS_HOST"] @@ -286,7 +286,7 @@ def set_env_variables(form): os.environ['SNOW_INSTANCE_NAME'] = form["SNOW_INSTANCE_NAME"] os.environ['SNOW_PASSWORD'] = form["SNOW_PASSWORD"] os.environ['SNOW_USERNAME'] = form["SNOW_USERNAME"] - os.environ['VERSION'] = form["VERSION"] + os.environ['VERSION'] = VERSION @admin.route('/dangerous/delete_all', methods=['POST']) From e50317f5019c213fb9c44a9630aa0cc38be4b08b Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:54:52 -0700 Subject: [PATCH 48/51] Feature/fix syntax error (#517) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py * Update updates.py * Update updates.py * Update settings.py * update to settings update without restart * Update run.py --- src/config_engine.py | 430 +++++++++++----------- src/run.py | 3 +- src/vr/__init__.py | 102 +++-- src/vr/admin/auth_functions.py | 4 +- src/vr/admin/email_alerts.py | 8 +- src/vr/admin/models.py | 9 +- src/vr/admin/routes/forgotpw.py | 3 +- src/vr/admin/routes/forgotun.py | 3 +- src/vr/admin/routes/login.py | 15 +- src/vr/admin/routes/logout.py | 6 +- src/vr/admin/routes/register.py | 3 +- src/vr/admin/routes/settings.py | 130 ++++--- src/vr/admin/routes/users.py | 3 +- src/vr/api/integrations/servicenow.py | 13 +- src/vr/api/vulns/jenkins_webhook.py | 6 +- src/vr/api/vulns/vulnerabilities.py | 3 +- src/vr/db_models/setup.py | 9 +- src/vr/db_models/setup_2.py | 4 +- src/vr/orchestration/web/pipeline_jobs.py | 7 +- src/vr/templates/admin/settings.html | 12 +- src/vr/vulns/web/findings.py | 3 +- src/vr/vulns/web/testing.py | 8 +- 22 files changed, 421 insertions(+), 363 deletions(-) diff --git a/src/config_engine.py b/src/config_engine.py index 07400d06..7ee859ad 100644 --- a/src/config_engine.py +++ b/src/config_engine.py @@ -14,98 +14,229 @@ from settings import SET_SNOW_INSTANCE_NAME, SET_SNOW_CLIENT_ID, SET_SNOW_CLIENT_SECRET, SET_SNOW_USERNAME, SET_SNOW_PASSWORD, SET_SNOW_CLIENT_SECRET_REF, SET_SNOW_PASSWORD_REF -VERSION = '0.1.0-beta' - -if os.getenv('AZURE_KEYVAULT_NAME'): - AZURE_KEYVAULT_NAME = os.getenv('AZURE_KEYVAULT_NAME') -else: - AZURE_KEYVAULT_NAME = SET_AZURE_KEYVAULT_NAME - -if os.getenv('AUTH_TYPE'): - AUTH_TYPE = os.getenv('AUTH_TYPE') -else: - AUTH_TYPE = SET_AUTH_TYPE - -if os.getenv('INSECURE_OAUTH'): - INSECURE_OAUTH = os.getenv('INSECURE_OAUTH') -else: - INSECURE_OAUTH = SET_INSECURE_OAUTH - -if INSECURE_OAUTH: - os.environ['AUTHLIB_INSECURE_TRANSPORT'] = '1' - -if os.getenv('APP_EXT_URL'): - APP_EXT_URL = os.getenv('APP_EXT_URL') -else: - APP_EXT_URL = SET_APP_EXT_URL - -if os.getenv('SMTP_HOST'): - SMTP_HOST = os.getenv('SMTP_HOST') -else: - SMTP_HOST = SET_SMTP_HOST - -if os.getenv('SMTP_USER'): - SMTP_USER = os.getenv('SMTP_USER') -else: - SMTP_USER = SET_SMTP_USER - -if os.getenv('SMTP_ADMIN_EMAIL'): - SMTP_ADMIN_EMAIL = os.getenv('SMTP_ADMIN_EMAIL') -else: - SMTP_ADMIN_EMAIL = SET_SMTP_ADMIN_EMAIL - -if os.getenv('LDAP_HOST'): - LDAP_HOST = os.getenv('LDAP_HOST') -else: - LDAP_HOST = SET_LDAP_HOST - -if os.getenv('LDAP_PORT'): - LDAP_PORT = os.getenv('LDAP_PORT') -else: - LDAP_PORT = SET_LDAP_PORT - -if os.getenv('LDAP_BASE_DN'): - LDAP_BASE_DN = os.getenv('LDAP_BASE_DN') -else: - LDAP_BASE_DN = SET_LDAP_BASE_DN - -if os.getenv('LDAP_USER_DN'): - LDAP_USER_DN = os.getenv('LDAP_USER_DN') -else: - LDAP_USER_DN = SET_LDAP_USER_DN - -if os.getenv('LDAP_GROUP_DN'): - LDAP_GROUP_DN = os.getenv('LDAP_GROUP_DN') -else: - LDAP_GROUP_DN = SET_LDAP_GROUP_DN - -if os.getenv('LDAP_USER_RDN_ATTR'): - LDAP_USER_RDN_ATTR = os.getenv('LDAP_USER_RDN_ATTR') -else: - LDAP_USER_RDN_ATTR = SET_LDAP_USER_RDN_ATTR - -if os.getenv('LDAP_USER_LOGIN_ATTR'): - LDAP_USER_LOGIN_ATTR = os.getenv('LDAP_USER_LOGIN_ATTR') -else: - LDAP_USER_LOGIN_ATTR = SET_LDAP_USER_LOGIN_ATTR - -if os.getenv('LDAP_BIND_USER_DN'): - LDAP_BIND_USER_DN = os.getenv('LDAP_BIND_USER_DN') -else: - LDAP_BIND_USER_DN = SET_LDAP_BIND_USER_DN - -if os.getenv('LDAP_BIND_USER_PASSWORD'): - LDAP_BIND_USER_PASSWORD = os.getenv('LDAP_BIND_USER_PASSWORD') -else: - LDAP_BIND_USER_PASSWORD = SET_LDAP_BIND_USER_PASSWORD +def getConfigs(config): + config['TEST_SETTING'] = 'set' + + config['VERSION'] = '0.1.0-beta' + + if os.getenv('AZURE_KEYVAULT_NAME'): + config['AZURE_KEYVAULT_NAME'] = os.getenv('AZURE_KEYVAULT_NAME') + else: + config['AZURE_KEYVAULT_NAME'] = SET_AZURE_KEYVAULT_NAME + + if os.getenv('AUTH_TYPE'): + config['AUTH_TYPE'] = os.getenv('AUTH_TYPE') + else: + config['AUTH_TYPE'] = SET_AUTH_TYPE + + if os.getenv('INSECURE_OAUTH'): + config['INSECURE_OAUTH'] = os.getenv('INSECURE_OAUTH') + else: + config['INSECURE_OAUTH'] = SET_INSECURE_OAUTH + + if config['INSECURE_OAUTH']: + os.environ['AUTHLIB_INSECURE_TRANSPORT'] = '1' + + if os.getenv('APP_EXT_URL'): + config['APP_EXT_URL'] = os.getenv('APP_EXT_URL') + else: + config['APP_EXT_URL'] = SET_APP_EXT_URL + + if os.getenv('SMTP_HOST'): + config['SMTP_HOST'] = os.getenv('SMTP_HOST') + else: + config['SMTP_HOST'] = SET_SMTP_HOST + + if os.getenv('SMTP_USER'): + config['SMTP_USER'] = os.getenv('SMTP_USER') + else: + config['SMTP_USER'] = SET_SMTP_USER + + if os.getenv('SMTP_ADMIN_EMAIL'): + config['SMTP_ADMIN_EMAIL'] = os.getenv('SMTP_ADMIN_EMAIL') + else: + config['SMTP_ADMIN_EMAIL'] = SET_SMTP_ADMIN_EMAIL + + if os.getenv('LDAP_HOST'): + config['LDAP_HOST'] = os.getenv('LDAP_HOST') + else: + config['LDAP_HOST'] = SET_LDAP_HOST + + if os.getenv('LDAP_PORT'): + config['LDAP_PORT'] = os.getenv('LDAP_PORT') + else: + config['LDAP_PORT'] = SET_LDAP_PORT + + if os.getenv('LDAP_BASE_DN'): + config['LDAP_BASE_DN'] = os.getenv('LDAP_BASE_DN') + else: + config['LDAP_BASE_DN'] = SET_LDAP_BASE_DN + + if os.getenv('LDAP_USER_DN'): + config['LDAP_USER_DN'] = os.getenv('LDAP_USER_DN') + else: + config['LDAP_USER_DN'] = SET_LDAP_USER_DN + + if os.getenv('LDAP_GROUP_DN'): + config['LDAP_GROUP_DN'] = os.getenv('LDAP_GROUP_DN') + else: + config['LDAP_GROUP_DN'] = SET_LDAP_GROUP_DN + + if os.getenv('LDAP_USER_RDN_ATTR'): + config['LDAP_USER_RDN_ATTR'] = os.getenv('LDAP_USER_RDN_ATTR') + else: + config['LDAP_USER_RDN_ATTR'] = SET_LDAP_USER_RDN_ATTR + + if os.getenv('LDAP_USER_LOGIN_ATTR'): + config['LDAP_USER_LOGIN_ATTR'] = os.getenv('LDAP_USER_LOGIN_ATTR') + else: + config['LDAP_USER_LOGIN_ATTR'] = SET_LDAP_USER_LOGIN_ATTR + + if os.getenv('LDAP_BIND_USER_DN'): + config['LDAP_BIND_USER_DN'] = os.getenv('LDAP_BIND_USER_DN') + else: + config['LDAP_BIND_USER_DN'] = SET_LDAP_BIND_USER_DN + + if os.getenv('LDAP_BIND_USER_PASSWORD'): + config['LDAP_BIND_USER_PASSWORD'] = os.getenv('LDAP_BIND_USER_PASSWORD') + else: + config['LDAP_BIND_USER_PASSWORD'] = SET_LDAP_BIND_USER_PASSWORD + + ## CORE Config Variables ## + if os.getenv('ENV'): + config['ENV'] = os.getenv('ENV') + else: + config['ENV'] = SET_ENV + + if config['ENV'] == 'prod': + if os.getenv('PROD_DB_URI_REF'): + config['PROD_DB_URI'] = KeyVaultManager(config).get_secret(os.getenv('PROD_DB_URI_REF')) + else: + config['PROD_DB_URI'] = KeyVaultManager(config).get_secret(SET_PROD_DB_URI_REF) + else: + config['PROD_DB_URI'] = SET_PROD_DB_URI + + if config['AUTH_TYPE'] == 'azuread': + if os.getenv('AZAD_CLIENT_ID'): + config['AZAD_CLIENT_ID'] = os.getenv('AZAD_CLIENT_ID') + else: + config['AZAD_CLIENT_ID'] = SET_AZAD_CLIENT_ID + if os.getenv('AZAD_CLIENT_SECRET'): + config['AZAD_CLIENT_SECRET'] = KeyVaultManager(config).get_secret(os.getenv('AZAD_CLIENT_SECRET')) + else: + config['AZAD_CLIENT_SECRET'] = KeyVaultManager(config).get_secret(SET_AZAD_CLIENT_SECRET) + if os.getenv('AZAD_AUTHORITY'): + config['AZAD_AUTHORITY'] = os.getenv('AZAD_AUTHORITY') + else: + config['AZAD_AUTHORITY'] = SET_AZAD_AUTHORITY + else: + config['AZAD_CLIENT_ID'] = "" + config['AZAD_CLIENT_SECRET'] = "" + config['AZAD_AUTHORITY'] = "" + + ## Email Variables ## + if config['ENV'] == 'prod': + if os.getenv('SMTP_PW_REF'): + config['SMTP_PASSWORD'] = KeyVaultManager(config).get_secret(os.getenv('SMTP_PW_REF')) + else: + config['SMTP_PASSWORD'] = KeyVaultManager(config).get_secret(SET_SMTP_PW_REF) + else: + config['SMTP_PASSWORD'] = SET_SMTP_PW + + ## + ## GitHub to Jenkins Webhook ## + if os.getenv('JENKINS_ENABLED'): + config['JENKINS_ENABLED'] = os.getenv('JENKINS_ENABLED') + else: + config['JENKINS_ENABLED'] = SET_JENKINS_ENABLED + if config['JENKINS_ENABLED'] == 'yes': + if config['ENV'] == 'prod': + if os.getenv('JENKINS_USER'): + config['JENKINS_USER'] = KeyVaultManager(config).get_secret(os.getenv('JENKINS_USER')) + else: + config['JENKINS_USER'] = KeyVaultManager(config).get_secret(SET_JENKINS_USER_REF) + if os.getenv('JENKINS_KEY'): + config['JENKINS_KEY'] = KeyVaultManager(config).get_secret(os.getenv('JENKINS_KEY')) + else: + config['JENKINS_KEY'] = KeyVaultManager(config).get_secret(SET_JENKINS_KEY_REF) + if os.getenv('JENKINS_TOKEN'): + config['JENKINS_TOKEN'] = KeyVaultManager(config).get_secret(os.getenv('JENKINS_TOKEN')) + else: + config['JENKINS_TOKEN'] = KeyVaultManager(config).get_secret(SET_JENKINS_TOKEN_REF) + else: + config['JENKINS_USER'] = SET_JENKINS_USER + config['JENKINS_KEY'] = SET_JENKINS_KEY + config['JENKINS_TOKEN'] = SET_JENKINS_TOKEN + + if os.getenv('JENKINS_PROJECT'): + config['JENKINS_PROJECT'] = os.getenv('JENKINS_PROJECT') + else: + config['JENKINS_PROJECT'] = SET_JENKINS_PROJECT + + if os.getenv('JENKINS_HOST'): + config['JENKINS_HOST'] = os.getenv('JENKINS_HOST') + else: + config['JENKINS_HOST'] = SET_JENKINS_HOST + + if os.getenv('JENKINS_STAGING_PROJECT'): + config['JENKINS_STAGING_PROJECT'] = os.getenv('JENKINS_STAGING_PROJECT') + else: + config['JENKINS_STAGING_PROJECT'] = SET_JENKINS_STAGING_PROJECT + else: + config['JENKINS_USER'] = "" + config['JENKINS_KEY'] = "" + config['JENKINS_TOKEN'] = "" + config['JENKINS_PROJECT'] = "" + config['JENKINS_HOST'] = "" + config['JENKINS_STAGING_PROJECT'] = "" + + ## ServiceNOW Integration + if os.getenv('SNOW_ENABLED'): + config['SNOW_ENABLED'] = os.getenv('SNOW_ENABLED') + else: + config['SNOW_ENABLED'] = SET_SNOW_ENABLED + if config['SNOW_ENABLED'] == 'yes': + if config['ENV'] == 'prod': + if os.getenv('SNOW_PASSWORD'): + config['SNOW_PASSWORD'] = KeyVaultManager(config).get_secret(os.getenv('SNOW_PASSWORD')) + else: + config['SNOW_PASSWORD'] = KeyVaultManager(config).get_secret(SET_SNOW_PASSWORD_REF) + if os.getenv('SNOW_CLIENT_SECRET'): + config['SNOW_CLIENT_SECRET'] = KeyVaultManager(config).get_secret(os.getenv('SNOW_CLIENT_SECRET')) + else: + config['SNOW_CLIENT_SECRET'] = KeyVaultManager(config).get_secret(SET_SNOW_CLIENT_SECRET_REF) + if os.getenv('SNOW_INSTANCE_NAME'): + config['SNOW_INSTANCE_NAME'] = os.getenv('SNOW_INSTANCE_NAME') + else: + config['SNOW_INSTANCE_NAME'] = SET_SNOW_INSTANCE_NAME + if os.getenv('SNOW_CLIENT_ID'): + config['SNOW_CLIENT_ID'] = os.getenv('SNOW_CLIENT_ID') + else: + config['SNOW_CLIENT_ID'] = SET_SNOW_CLIENT_ID + if os.getenv('SNOW_USERNAME'): + config['SNOW_USERNAME'] = os.getenv('SNOW_USERNAME') + else: + config['SNOW_USERNAME'] = SET_SNOW_USERNAME + else: + config['SNOW_PASSWORD'] = SET_SNOW_PASSWORD + config['SNOW_CLIENT_SECRET'] = SET_SNOW_CLIENT_SECRET + config['SNOW_INSTANCE_NAME'] = SET_SNOW_INSTANCE_NAME + config['SNOW_CLIENT_ID'] = SET_SNOW_CLIENT_ID + config['SNOW_USERNAME'] = SET_SNOW_USERNAME + else: + config['SNOW_PASSWORD'] = "" + config['SNOW_CLIENT_SECRET'] = "" + config['SNOW_INSTANCE_NAME'] = "" + config['SNOW_CLIENT_ID'] = "" + config['SNOW_USERNAME'] = "" class KeyVaultManager(object): - def __init__(self): + def __init__(self, config=None): if os.getenv('AZURE_KEYVAULT_NAME'): key_vault_uri = f"https://{os.getenv('AZURE_KEYVAULT_NAME')}.vault.azure.net" else: - key_vault_uri = f"https://{AZURE_KEYVAULT_NAME}.vault.azure.net" + key_vault_uri = f"https://{config['AZURE_KEYVAULT_NAME']}.vault.azure.net" if os.getenv('AZURE_AUTH_METHOD'): if os.getenv('AZURE_AUTH_METHOD') == 'env': self.credential = EnvironmentCredential( @@ -162,130 +293,3 @@ def delete_cert(self, secret_name): return deleted_secret -## CORE Config Variables ## -if os.getenv('ENV'): - ENV = os.getenv('ENV') -else: - ENV = SET_ENV - -if ENV == 'prod': - if os.getenv('PROD_DB_URI_REF'): - PROD_DB_URI = KeyVaultManager().get_secret(os.getenv('PROD_DB_URI_REF')) - else: - PROD_DB_URI = KeyVaultManager().get_secret(SET_PROD_DB_URI_REF) -else: - PROD_DB_URI = SET_PROD_DB_URI - -if AUTH_TYPE == 'azuread': - if os.getenv('AZAD_CLIENT_ID'): - AZAD_CLIENT_ID = os.getenv('AZAD_CLIENT_ID') - else: - AZAD_CLIENT_ID = SET_AZAD_CLIENT_ID - if os.getenv('AZAD_CLIENT_SECRET'): - AZAD_CLIENT_SECRET = KeyVaultManager().get_secret(os.getenv('AZAD_CLIENT_SECRET')) - else: - AZAD_CLIENT_SECRET = KeyVaultManager().get_secret(SET_AZAD_CLIENT_SECRET) - if os.getenv('AZAD_AUTHORITY'): - AZAD_AUTHORITY = os.getenv('AZAD_AUTHORITY') - else: - AZAD_AUTHORITY = SET_AZAD_AUTHORITY -else: - AZAD_CLIENT_ID = "" - AZAD_CLIENT_SECRET = "" - AZAD_AUTHORITY = "" - -## Email Variables ## -if ENV == 'prod': - if os.getenv('SMTP_PW_REF'): - SMTP_PASSWORD = KeyVaultManager().get_secret(os.getenv('SMTP_PW_REF')) - else: - SMTP_PASSWORD = KeyVaultManager().get_secret(SET_SMTP_PW_REF) -else: - SMTP_PASSWORD = SET_SMTP_PW - -## -## GitHub to Jenkins Webhook ## -if os.getenv('JENKINS_ENABLED'): - JENKINS_ENABLED = os.getenv('JENKINS_ENABLED') -else: - JENKINS_ENABLED = SET_JENKINS_ENABLED -if JENKINS_ENABLED == 'yes': - if ENV == 'prod': - if os.getenv('JENKINS_USER'): - JENKINS_USER = KeyVaultManager().get_secret(os.getenv('JENKINS_USER')) - else: - JENKINS_USER = KeyVaultManager().get_secret(SET_JENKINS_USER_REF) - if os.getenv('JENKINS_KEY'): - JENKINS_KEY = KeyVaultManager().get_secret(os.getenv('JENKINS_KEY')) - else: - JENKINS_KEY = KeyVaultManager().get_secret(SET_JENKINS_KEY_REF) - if os.getenv('JENKINS_TOKEN'): - JENKINS_TOKEN = KeyVaultManager().get_secret(os.getenv('JENKINS_TOKEN')) - else: - JENKINS_TOKEN = KeyVaultManager().get_secret(SET_JENKINS_TOKEN_REF) - else: - JENKINS_USER = SET_JENKINS_USER - JENKINS_KEY = SET_JENKINS_KEY - JENKINS_TOKEN = SET_JENKINS_TOKEN - - if os.getenv('JENKINS_PROJECT'): - JENKINS_PROJECT = os.getenv('JENKINS_PROJECT') - else: - JENKINS_PROJECT = SET_JENKINS_PROJECT - - if os.getenv('JENKINS_HOST'): - JENKINS_HOST = os.getenv('JENKINS_HOST') - else: - JENKINS_HOST = SET_JENKINS_HOST - - if os.getenv('JENKINS_STAGING_PROJECT'): - JENKINS_STAGING_PROJECT = os.getenv('JENKINS_STAGING_PROJECT') - else: - JENKINS_STAGING_PROJECT = SET_JENKINS_STAGING_PROJECT -else: - JENKINS_USER = "" - JENKINS_KEY = "" - JENKINS_TOKEN = "" - JENKINS_PROJECT = "" - JENKINS_HOST = "" - JENKINS_STAGING_PROJECT = "" - -## ServiceNOW Integration -if os.getenv('SNOW_ENABLED'): - SNOW_ENABLED = os.getenv('SNOW_ENABLED') -else: - SNOW_ENABLED = SET_SNOW_ENABLED -if SNOW_ENABLED == 'yes': - if ENV == 'prod': - if os.getenv('SNOW_PASSWORD'): - SNOW_PASSWORD = KeyVaultManager().get_secret(os.getenv('SNOW_PASSWORD')) - else: - SNOW_PASSWORD = KeyVaultManager().get_secret(SET_SNOW_PASSWORD_REF) - if os.getenv('SNOW_CLIENT_SECRET'): - SNOW_CLIENT_SECRET = KeyVaultManager().get_secret(os.getenv('SNOW_CLIENT_SECRET')) - else: - SNOW_CLIENT_SECRET = KeyVaultManager().get_secret(SET_SNOW_CLIENT_SECRET_REF) - if os.getenv('SNOW_INSTANCE_NAME'): - SNOW_INSTANCE_NAME = os.getenv('SNOW_INSTANCE_NAME') - else: - SNOW_INSTANCE_NAME = SET_SNOW_INSTANCE_NAME - if os.getenv('SNOW_CLIENT_ID'): - SNOW_CLIENT_ID = os.getenv('SNOW_CLIENT_ID') - else: - SNOW_CLIENT_ID = SET_SNOW_CLIENT_ID - if os.getenv('SNOW_USERNAME'): - SNOW_USERNAME = os.getenv('SNOW_USERNAME') - else: - SNOW_USERNAME = SET_SNOW_USERNAME - else: - SNOW_PASSWORD = SET_SNOW_PASSWORD - SNOW_CLIENT_SECRET = SET_SNOW_CLIENT_SECRET - SNOW_INSTANCE_NAME = SET_SNOW_INSTANCE_NAME - SNOW_CLIENT_ID = SET_SNOW_CLIENT_ID - SNOW_USERNAME = SET_SNOW_USERNAME -else: - SNOW_PASSWORD = "" - SNOW_CLIENT_SECRET = "" - SNOW_INSTANCE_NAME = "" - SNOW_CLIENT_ID = "" - SNOW_USERNAME = "" diff --git a/src/run.py b/src/run.py index 7f0f15c7..b67be678 100644 --- a/src/run.py +++ b/src/run.py @@ -4,10 +4,9 @@ import datetime import os from vr.admin.oauth2 import config_oauth -from config_engine import ENV, INSECURE_OAUTH -if ENV == 'test' or INSECURE_OAUTH: +if app.config['ENV'] == 'test' or app.config['INSECURE_OAUTH']: os.environ['AUTHLIB_INSECURE_TRANSPORT'] = '1' else: os.environ['AUTHLIB_INSECURE_TRANSPORT'] = '0' diff --git a/src/vr/__init__.py b/src/vr/__init__.py index 42a86ea1..2f3ccf73 100644 --- a/src/vr/__init__.py +++ b/src/vr/__init__.py @@ -1,8 +1,6 @@ import datetime import requests -from config_engine import ENV, PROD_DB_URI, AUTH_TYPE, APP_EXT_URL, LDAP_HOST, LDAP_PORT, LDAP_BASE_DN, \ - LDAP_USER_DN, LDAP_GROUP_DN, LDAP_USER_RDN_ATTR, LDAP_USER_LOGIN_ATTR, LDAP_BIND_USER_DN, LDAP_BIND_USER_PASSWORD, \ - AZAD_CLIENT_ID, AZAD_CLIENT_SECRET, AZAD_AUTHORITY, JENKINS_USER, JENKINS_ENABLED +from config_engine import getConfigs from flask import Flask from flask_bootstrap import Bootstrap from flask_login import LoginManager @@ -11,8 +9,7 @@ from flask_wtf.csrf import CSRFProtect from flaskext.markdown import Markdown from vr.db_models.setup import _init_db -if AUTH_TYPE == 'ldap': - from flask_ldap3_login import LDAP3LoginManager + import base64 import logging import sys @@ -31,48 +28,41 @@ from requests.auth import HTTPBasicAuth from vr.db_models.updates import createNewTables -if AUTH_TYPE == 'azuread': + +app = Flask(__name__) + +getConfigs(app.config) + +if app.config['AUTH_TYPE'] == 'azuread': from flask_session import Session import msal from flask import session, url_for +if app.config['AUTH_TYPE'] == 'ldap': + from flask_ldap3_login import LDAP3LoginManager -app = Flask(__name__) moment = Moment(app) Markdown(app) csrf = CSRFProtect(app) -app.config['APP_EXT_URL'] = APP_EXT_URL - -app.config['RUNTIME_ENV'] = ENV +app.config['RUNTIME_ENV'] = app.config['ENV'] if app.config['RUNTIME_ENV'] == 'test': DB_URI = 'sqlite:///database.db' import sqlite3 else: - DB_URI = PROD_DB_URI + DB_URI = app.config['PROD_DB_URI'] import mysql.connector app.config['SQLALCHEMY_DATABASE_URI'] = DB_URI app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False -if AUTH_TYPE == 'ldap': - # LDAP Configuration - app.config['LDAP_HOST'] = LDAP_HOST - app.config['LDAP_PORT'] = LDAP_PORT - app.config['LDAP_BASE_DN'] = LDAP_BASE_DN - app.config['LDAP_USER_DN'] = LDAP_USER_DN - app.config['LDAP_GROUP_DN'] = LDAP_GROUP_DN - app.config['LDAP_USER_RDN_ATTR'] = LDAP_USER_RDN_ATTR - app.config['LDAP_USER_LOGIN_ATTR'] = LDAP_USER_LOGIN_ATTR - app.config['LDAP_BIND_USER_DN'] = LDAP_BIND_USER_DN - app.config['LDAP_BIND_USER_PASSWORD'] = LDAP_BIND_USER_PASSWORD - +if app.config['AUTH_TYPE'] == 'ldap': # Flask-LDAP3-Login Manager ldap_manager = LDAP3LoginManager(app) -elif AUTH_TYPE == 'azuread': - app.config['CLIENT_ID'] = AZAD_CLIENT_ID - app.config['CLIENT_SECRET'] = AZAD_CLIENT_SECRET - app.config['AUTHORITY'] = AZAD_AUTHORITY +elif app.config['AUTH_TYPE'] == 'azuread': + app.config['CLIENT_ID'] = app.config['AZAD_CLIENT_ID'] + app.config['CLIENT_SECRET'] = app.config['AZAD_CLIENT_SECRET'] + app.config['AUTHORITY'] = app.config['AZAD_AUTHORITY'] app.config['REDIRECT_PATH'] = "/getAToken" app.config['ENDPOINT'] = 'https://graph.microsoft.com/v1.0/me/memberOf' app.config['SCOPE'] = ["User.ReadBasic.All", "Group.Read.All", "Application.Read.All"] @@ -151,7 +141,7 @@ def _get_token_from_cache(scope=None): app.register_blueprint(api) bootstrap = Bootstrap(app) -if AUTH_TYPE == 'local' or AUTH_TYPE == 'azuread': +if app.config['AUTH_TYPE'] == 'local' or app.config['AUTH_TYPE'] == 'azuread': login_manager.init_app(app) login_manager.login_view = 'admin.login' @@ -162,9 +152,10 @@ def _get_token_from_cache(scope=None): app.logger.addHandler(stdout_handler) + @app.template_filter('format_datetime') def format_datetime(value): - if ENV == 'test': + if app.config['ENV'] == 'test': try: formatted = datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f") except: @@ -185,6 +176,8 @@ def base64encode(value): createNewTables(app) ## Cronjob-like tasks section ## + + def train_model_every_six_hours(): scheduler = BackgroundScheduler() scheduler.add_job(train_model, 'interval', hours=6) @@ -220,6 +213,53 @@ def connect_to_db(): cur = db.cursor() return cur, db +def getPersistentConfig(): + try: + cur, db = connect_to_db() + sql = 'SELECT * FROM AppConfig WHERE 1=1' + cur.execute(sql) + row = cur.fetchone() + if row[2]: + app.config['APP_EXT_URL'] = row[3] + app.config['AUTH_TYPE'] = row[4] + app.config['AZAD_AUTHORITY'] = row[5] + app.config['AZAD_CLIENT_ID'] = row[6] + app.config['AZAD_CLIENT_SECRET'] = row[7] + app.config['AZURE_KEYVAULT_NAME'] = row[8] + app.config['ENV'] = row[9] + app.config['INSECURE_OAUTH'] = row[10] + app.config['JENKINS_ENABLED'] = row[37] + app.config['JENKINS_HOST'] = row[11] + app.config['JENKINS_KEY'] = row[12] + app.config['JENKINS_PROJECT'] = row[13] + app.config['JENKINS_STAGING_PROJECT'] = row[14] + app.config['JENKINS_TOKEN'] = row[15] + app.config['JENKINS_USER'] = row[16] + app.config['LDAP_BASE_DN'] = row[17] + app.config['LDAP_BIND_USER_DN'] = row[18] + app.config['LDAP_BIND_USER_PASSWORD'] = row[19] + app.config['LDAP_GROUP_DN'] = row[20] + app.config['LDAP_HOST'] = row[21] + app.config['LDAP_PORT'] = row[22] + app.config['LDAP_USER_DN'] = row[23] + app.config['LDAP_USER_LOGIN_ATTR'] = row[24] + app.config['LDAP_USER_RDN_ATTR'] = row[25] + app.config['PROD_DB_URI'] = row[26] + app.config['SMTP_ADMIN_EMAIL'] = row[27] + app.config['SMTP_HOST'] = row[28] + app.config['SMTP_PASSWORD'] = row[29] + app.config['SMTP_USER'] = row[30] + app.config['SNOW_ENABLED'] = row[38] + app.config['SNOW_CLIENT_ID'] = row[31] + app.config['SNOW_CLIENT_SECRET'] = row[32] + app.config['SNOW_INSTANCE_NAME'] = row[33] + app.config['SNOW_PASSWORD'] = row[34] + app.config['SNOW_USERNAME'] = row[35] + app.config['VERSION'] = row[36] + except: + print('AppConfig Database table is either unreachable or not setup.') + +getPersistentConfig() def train_model(): try: @@ -337,7 +377,7 @@ def rsa_long_decrypt(priv_obj, msg, length=256): def get_jenkins_data(): - user_check = JENKINS_USER + user_check = app.config['JENKINS_USER'] if user_check != 'changeme': app.logger.info('Getting Jenkins Data') cur, db = connect_to_db() @@ -452,5 +492,5 @@ def get_jenkins_data(): # Call the Jobs Here # train_model_every_six_hours() -if JENKINS_ENABLED == 'yes': +if app.config['JENKINS_ENABLED'] == 'yes': get_jenkins_data_every_hour() diff --git a/src/vr/admin/auth_functions.py b/src/vr/admin/auth_functions.py index 20ddf7ca..885020c6 100644 --- a/src/vr/admin/auth_functions.py +++ b/src/vr/admin/auth_functions.py @@ -1,7 +1,7 @@ import jwt from time import time from vr.functions.mysql_db import connect_to_db -from config_engine import ENV +from vr import app # Error handler @@ -49,7 +49,7 @@ def create_api_key(user_id, otp_secret, expires_in=2592000): def verify_api_key(token): try: cur, db = connect_to_db() - if ENV == 'test': + if app.config['ENV'] == 'test': sql = 'SELECT oc.user_id, u.is_admin FROM oauth2_client oc JOIN oauth2_token ot ON oc.client_id=ot.client_id JOIN User u ON oc.user_id=u.id WHERE ot.id=?' else: sql = 'SELECT oc.user_id, u.is_admin FROM oauth2_client oc JOIN oauth2_token ot ON oc.client_id=ot.client_id JOIN User u ON oc.user_id=u.id WHERE ot.id=%s' diff --git a/src/vr/admin/email_alerts.py b/src/vr/admin/email_alerts.py index a1c48da3..698b5138 100644 --- a/src/vr/admin/email_alerts.py +++ b/src/vr/admin/email_alerts.py @@ -1,7 +1,7 @@ import smtplib from email.mime import multipart from email.mime import text as mimetext -from config_engine import SMTP_HOST, SMTP_USER, SMTP_PASSWORD, SMTP_ADMIN_EMAIL +from vr import app def send_email(msg_fromaddr, msg_toaddr, msg_subject, msg_body): @@ -11,9 +11,9 @@ def send_email(msg_fromaddr, msg_toaddr, msg_subject, msg_body): msg['To'] = msg_toaddr msg['Subject'] = msg_subject msg.attach(mimetext.MIMEText(message, 'html')) - server = smtplib.SMTP(SMTP_HOST) + server = smtplib.SMTP(app.config['SMTP_HOST']) server.starttls() - server.login(SMTP_USER, SMTP_PASSWORD) + server.login(app.config['SMTP_USER'], app.config['SMTP_PASSWORD']) server.ehlo() text = msg.as_string() server.sendmail(msg_fromaddr, msg_toaddr, text) @@ -24,7 +24,7 @@ def send_registration_email(ext_url, username, first_name, last_name, token, ema msg_subject = "SecuSphere User Registration" msg_body = generate_registration_msg(ext_url, username, first_name, last_name, token) try: - send_email(SMTP_ADMIN_EMAIL, email_to, msg_subject, msg_body) + send_email(app.config['SMTP_ADMIN_EMAIL'], email_to, msg_subject, msg_body) except: return 'error' diff --git a/src/vr/admin/models.py b/src/vr/admin/models.py index 89785772..d5d5e4ad 100644 --- a/src/vr/admin/models.py +++ b/src/vr/admin/models.py @@ -3,7 +3,7 @@ from flask_login import UserMixin from vr import db, app from vr.functions.mysql_db import connect_to_db -from datetime import datetime, timedelta +from datetime import datetime import jwt from vr.admin.helper_functions import hash_password,verify_password from vr.admin.functions import db_connection_handler @@ -17,8 +17,7 @@ OAuth2AuthorizationCodeMixin, OAuth2TokenMixin, ) -from config_engine import AUTH_TYPE -if AUTH_TYPE == 'ldap': +if app.config['AUTH_TYPE'] == 'ldap': from vr import ldap_manager if app.config['RUNTIME_ENV'] == 'test': @@ -190,11 +189,11 @@ def verify_username_token(self, token, given_id): else: return -if AUTH_TYPE == 'local' or AUTH_TYPE == 'azuread': +if app.config['AUTH_TYPE'] == 'local' or app.config['AUTH_TYPE'] == 'azuread': @login_manager.user_loader def load_user(id): return User.query.get(int(id)) -elif AUTH_TYPE == 'ldap': +elif app.config['AUTH_TYPE'] == 'ldap': # User Loader for LDAP @login_manager.user_loader def load_user(user_id): diff --git a/src/vr/admin/routes/forgotpw.py b/src/vr/admin/routes/forgotpw.py index 2d12cd59..f6d50228 100644 --- a/src/vr/admin/routes/forgotpw.py +++ b/src/vr/admin/routes/forgotpw.py @@ -6,7 +6,6 @@ from vr.admin.email_alerts import send_email, generate_evnt_msg from vr.functions.timefunctions import return_datetime_now from vr.admin.helper_functions import hash_password -from config_engine import SMTP_ADMIN_EMAIL from vr.admin.functions import db_connection_handler @@ -32,7 +31,7 @@ def forgotpw(): action_list = [action] st = 'n' msg_body = generate_evnt_msg(msg_subject, now, evt_list, action_list, st) - msg_fromaddr = SMTP_ADMIN_EMAIL + msg_fromaddr = app.config['SMTP_ADMIN_EMAIL'] try: send_email(msg_fromaddr, email, msg_subject, msg_body) warnmsg = ('pwresetemail', 'success') diff --git a/src/vr/admin/routes/forgotun.py b/src/vr/admin/routes/forgotun.py index 80c52425..d8a0d2fa 100644 --- a/src/vr/admin/routes/forgotun.py +++ b/src/vr/admin/routes/forgotun.py @@ -5,7 +5,6 @@ from vr.admin.models import User, LoginForm from vr.admin.email_alerts import send_email, generate_evnt_msg from vr.functions.timefunctions import return_datetime_now -from config_engine import SMTP_ADMIN_EMAIL NAV_CAT= { "name": "Admin", "url": "admin.admin_dashboard"} @@ -28,7 +27,7 @@ def forgotun(): action_list = [action] st = 'n' msg_body = generate_evnt_msg(msg_subject,now,evt_list,action_list,st) - msg_fromaddr = SMTP_ADMIN_EMAIL + msg_fromaddr = app.config['SMTP_ADMIN_EMAIL'] try: send_email(msg_fromaddr, email, msg_subject, msg_body) warnmsg = ('pwresetemail', 'success') diff --git a/src/vr/admin/routes/login.py b/src/vr/admin/routes/login.py index 9eed58a7..d7f641f1 100644 --- a/src/vr/admin/routes/login.py +++ b/src/vr/admin/routes/login.py @@ -8,10 +8,9 @@ from vr.admin.models import User, LoginForm, AuthAttempts, AppConfig from vr.admin.functions import _auth_user, _entity_permissions_filter, _entity_page_permissions_filter, check_lockout, log_failed_attempt from vr.admin.functions import db_connection_handler -from config_engine import AUTH_TYPE -if AUTH_TYPE == 'ldap': +if app.config['AUTH_TYPE'] == 'ldap': from flask_ldap3_login.forms import LDAPLoginForm -elif AUTH_TYPE == 'azuread': +elif app.config['AUTH_TYPE'] == 'azuread': import requests import msal from vr import _build_auth_code_flow, _load_cache, _save_cache, _build_msal_app, _get_token_from_cache @@ -35,7 +34,7 @@ def login(): return redirect(url_for('admin.register')) ad_auth_url = None warnmsg = '' - if AUTH_TYPE == 'local': + if app.config['AUTH_TYPE'] == 'local': if current_user.is_authenticated: flash('You are already logged in.', 'danger') return redirect(url_for('assets.all_applications')) @@ -60,7 +59,7 @@ def login(): mfa_password = resp[2] # attempt to log the user in return _login_attempt(user, username, password, userid, form, mfa_password) - elif AUTH_TYPE == 'ldap': + elif app.config['AUTH_TYPE'] == 'ldap': form = LDAPLoginForm() if form.validate_on_submit(): # Log the user in @@ -71,15 +70,15 @@ def login(): # Print the form errors print("Form validation failed with errors:", form.errors) return render_template(LDAP_LOGIN_TEMPLATE, form=form, errors=form.errors) - elif AUTH_TYPE == 'azuread': + elif app.config['AUTH_TYPE'] == 'azuread': form = LoginForm(request.form) session["flow"] = _build_auth_code_flow(scopes=app.config['SCOPE']) ad_auth_url = session["flow"]["auth_uri"] if form.errors: warnmsg = (form.errors, 'danger') - return render_template(LOGIN_TEMPLATE, form=form, warnmsg=warnmsg, auth_type=AUTH_TYPE, auth_url=ad_auth_url) + return render_template(LOGIN_TEMPLATE, form=form, warnmsg=warnmsg, auth_type=app.config['AUTH_TYPE'], auth_url=ad_auth_url) -if AUTH_TYPE == 'azuread': +if app.config['AUTH_TYPE'] == 'azuread': @app.route(app.config['REDIRECT_PATH']) # Its absolute URL must match your app's redirect_uri set in AAD def authorized(): try: diff --git a/src/vr/admin/routes/logout.py b/src/vr/admin/routes/logout.py index 218cd357..781302c9 100644 --- a/src/vr/admin/routes/logout.py +++ b/src/vr/admin/routes/logout.py @@ -1,20 +1,20 @@ from flask_login import logout_user, login_required from flask import session, redirect, url_for from vr.admin import admin -from config_engine import AUTH_TYPE +from vr import app NAV_CAT= { "name": "Admin", "url": "admin.admin_dashboard"} -if AUTH_TYPE == 'local': +if app.config['AUTH_TYPE'] == 'local': @admin.route('/logout') @login_required def logout(): logout_user() del session['username'] return redirect(url_for('admin.login')) -elif AUTH_TYPE == 'azuread': +elif app.config['AUTH_TYPE'] == 'azuread': @admin.route('/logout') def logout(): logout_user() diff --git a/src/vr/admin/routes/register.py b/src/vr/admin/routes/register.py index 38f0b3f8..1f6f38a8 100644 --- a/src/vr/admin/routes/register.py +++ b/src/vr/admin/routes/register.py @@ -13,7 +13,6 @@ from vr.admin.helper_functions import hash_password from vr.admin.email_alerts import send_registration_email from vr.functions.initial_setup import setup_core_db_tables, generate_key_pair -from config_engine import ENV from vr.db_models.setup_2 import _init_db @@ -115,7 +114,7 @@ def register_submit(): _init_db(db=db) generate_key_pair() - setup_core_db_tables(ENV) + setup_core_db_tables(app.config['ENV']) admin_role = UserRoles.query.filter_by(name='Admin').first() ura = UserRoleAssignments(user_id=user.id, role_id=admin_role.id) db.session.add(ura) diff --git a/src/vr/admin/routes/settings.py b/src/vr/admin/routes/settings.py index fea3e3b5..532ae0d5 100644 --- a/src/vr/admin/routes/settings.py +++ b/src/vr/admin/routes/settings.py @@ -5,16 +5,8 @@ # Start of Entity-specific Imports from vr.admin import admin from vr.admin.functions import _auth_user, check_menu_tour_init -from config_engine import ENV, PROD_DB_URI, AUTH_TYPE, APP_EXT_URL, LDAP_HOST, LDAP_PORT, LDAP_BASE_DN, \ - LDAP_USER_DN, LDAP_GROUP_DN, LDAP_USER_RDN_ATTR, LDAP_USER_LOGIN_ATTR, LDAP_BIND_USER_DN, LDAP_BIND_USER_PASSWORD, \ - AZAD_CLIENT_ID, AZAD_CLIENT_SECRET, AZAD_AUTHORITY, JENKINS_USER, AZURE_KEYVAULT_NAME, INSECURE_OAUTH, \ - JENKINS_HOST, JENKINS_KEY, JENKINS_PROJECT, JENKINS_STAGING_PROJECT, JENKINS_TOKEN, SMTP_ADMIN_EMAIL, \ - SMTP_HOST, SMTP_PASSWORD, SMTP_USER, SNOW_CLIENT_ID, SNOW_CLIENT_SECRET, SNOW_INSTANCE_NAME, SNOW_PASSWORD, \ - SNOW_USERNAME, VERSION, JENKINS_ENABLED, SNOW_ENABLED -from flask_sqlalchemy import SQLAlchemy from vr.admin.models import AppConfig from vr.admin.functions import db_connection_handler -from sqlalchemy import text NAV = { @@ -44,7 +36,7 @@ def settings(): AppConfig.AZAD_CLIENT_ID: all["AZAD_CLIENT_ID"], AppConfig.AZAD_CLIENT_SECRET: all["AZAD_CLIENT_SECRET"], AppConfig.AZURE_KEYVAULT_NAME: all["AZURE_KEYVAULT_NAME"], - AppConfig.ENV: ENV, + AppConfig.ENV: app.config['ENV'], AppConfig.INSECURE_OAUTH: all["INSECURE_OAUTH"], AppConfig.JENKINS_HOST: all["JENKINS_HOST"], AppConfig.JENKINS_KEY: all["JENKINS_KEY"], @@ -71,7 +63,7 @@ def settings(): AppConfig.SNOW_INSTANCE_NAME: all["SNOW_INSTANCE_NAME"], AppConfig.SNOW_PASSWORD: all["SNOW_PASSWORD"], AppConfig.SNOW_USERNAME: all["SNOW_USERNAME"], - AppConfig.VERSION: VERSION, + AppConfig.VERSION: app.config['VERSION'], } if not app_config.settings_initialized: update_json[AppConfig.settings_initialized] = True @@ -88,7 +80,7 @@ def settings(): "AZAD_CLIENT_ID": all["AZAD_CLIENT_ID"], "AZAD_CLIENT_SECRET": all["AZAD_CLIENT_SECRET"], "AZURE_KEYVAULT_NAME": all["AZURE_KEYVAULT_NAME"], - "ENV": ENV, + "ENV": app.config['ENV'], "INSECURE_OAUTH": all["INSECURE_OAUTH"], "JENKINS_HOST": all["JENKINS_HOST"], "JENKINS_KEY": all["JENKINS_KEY"], @@ -115,7 +107,7 @@ def settings(): "SNOW_INSTANCE_NAME": all["SNOW_INSTANCE_NAME"], "SNOW_PASSWORD": all["SNOW_PASSWORD"], "SNOW_USERNAME": all["SNOW_USERNAME"], - "VERSION": VERSION, + "VERSION": app.config['VERSION'], } else: app_config = AppConfig.query.first() @@ -160,42 +152,42 @@ def settings(): } else: current_settings = { - "JENKINS_ENABLED": JENKINS_ENABLED, - "SNOW_ENABLED": SNOW_ENABLED, - "APP_EXT_URL": APP_EXT_URL, - "AUTH_TYPE": AUTH_TYPE, - "AZAD_AUTHORITY":AZAD_AUTHORITY, - "AZAD_CLIENT_ID": AZAD_CLIENT_ID, - "AZAD_CLIENT_SECRET": AZAD_CLIENT_SECRET, - "AZURE_KEYVAULT_NAME": AZURE_KEYVAULT_NAME, - "ENV": ENV, - "INSECURE_OAUTH": INSECURE_OAUTH, - "JENKINS_HOST": JENKINS_HOST, - "JENKINS_KEY": JENKINS_KEY, - "JENKINS_PROJECT": JENKINS_PROJECT, - "JENKINS_STAGING_PROJECT": JENKINS_STAGING_PROJECT, - "JENKINS_USER": JENKINS_USER, - "JENKINS_TOKEN": JENKINS_TOKEN, - "LDAP_BASE_DN": LDAP_BASE_DN, - "LDAP_BIND_USER_DN": LDAP_BIND_USER_DN, - "LDAP_BIND_USER_PASSWORD": LDAP_BIND_USER_PASSWORD, - "LDAP_GROUP_DN": LDAP_GROUP_DN, - "LDAP_HOST": LDAP_HOST, - "LDAP_PORT": LDAP_PORT, - "LDAP_USER_DN": LDAP_USER_DN, - "LDAP_USER_LOGIN_ATTR": LDAP_USER_LOGIN_ATTR, - "LDAP_USER_RDN_ATTR": LDAP_USER_RDN_ATTR, - "PROD_DB_URI": PROD_DB_URI, - "SMTP_ADMIN_EMAIL": SMTP_ADMIN_EMAIL, - "SMTP_HOST": SMTP_HOST, - "SMTP_USER": SMTP_USER, - "SMTP_PASSWORD": SMTP_PASSWORD, - "SNOW_CLIENT_ID": SNOW_CLIENT_ID, - "SNOW_CLIENT_SECRET": SNOW_CLIENT_SECRET, - "SNOW_INSTANCE_NAME": SNOW_INSTANCE_NAME, - "SNOW_USERNAME": SNOW_USERNAME, - "SNOW_PASSWORD": SNOW_PASSWORD, - "VERSION": VERSION, + "JENKINS_ENABLED": app.config['JENKINS_ENABLED'], + "SNOW_ENABLED": app.config['SNOW_ENABLED'], + "APP_EXT_URL": app.config['APP_EXT_URL'], + "AUTH_TYPE": app.config['AUTH_TYPE'], + "AZAD_AUTHORITY": app.config['AZAD_AUTHORITY'], + "AZAD_CLIENT_ID": app.config['AZAD_CLIENT_ID'], + "AZAD_CLIENT_SECRET": app.config['AZAD_CLIENT_SECRET'], + "AZURE_KEYVAULT_NAME": app.config['AZURE_KEYVAULT_NAME'], + "ENV": app.config['ENV'], + "INSECURE_OAUTH": app.config['INSECURE_OAUTH'], + "JENKINS_HOST": app.config['JENKINS_HOST'], + "JENKINS_KEY": app.config['JENKINS_KEY'], + "JENKINS_PROJECT": app.config['JENKINS_PROJECT'], + "JENKINS_STAGING_PROJECT": app.config['JENKINS_STAGING_PROJECT'], + "JENKINS_USER": app.config['JENKINS_USER'], + "JENKINS_TOKEN": app.config['JENKINS_TOKEN'], + "LDAP_BASE_DN": app.config['LDAP_BASE_DN'], + "LDAP_BIND_USER_DN": app.config['LDAP_BIND_USER_DN'], + "LDAP_BIND_USER_PASSWORD": app.config['LDAP_BIND_USER_PASSWORD'], + "LDAP_GROUP_DN": app.config['LDAP_GROUP_DN'], + "LDAP_HOST": app.config['LDAP_HOST'], + "LDAP_PORT": app.config['LDAP_PORT'], + "LDAP_USER_DN": app.config['LDAP_USER_DN'], + "LDAP_USER_LOGIN_ATTR": app.config['LDAP_USER_LOGIN_ATTR'], + "LDAP_USER_RDN_ATTR": app.config['LDAP_USER_RDN_ATTR'], + "PROD_DB_URI": app.config['PROD_DB_URI'], + "SMTP_ADMIN_EMAIL": app.config['SMTP_ADMIN_EMAIL'], + "SMTP_HOST": app.config['SMTP_HOST'], + "SMTP_USER": app.config['SMTP_USER'], + "SMTP_PASSWORD": app.config['SMTP_PASSWORD'], + "SNOW_CLIENT_ID": app.config['SNOW_CLIENT_ID'], + "SNOW_CLIENT_SECRET": app.config['SNOW_CLIENT_SECRET'], + "SNOW_INSTANCE_NAME": app.config['SNOW_INSTANCE_NAME'], + "SNOW_USERNAME": app.config['SNOW_USERNAME'], + "SNOW_PASSWORD": app.config['SNOW_PASSWORD'], + "VERSION": app.config['VERSION'], } cat_general = [ 'APP_EXT_URL', @@ -257,7 +249,7 @@ def set_env_variables(form): os.environ['AZAD_CLIENT_ID'] = form["AZAD_CLIENT_ID"] os.environ['AZAD_CLIENT_SECRET'] = form["AZAD_CLIENT_SECRET"] os.environ['AZURE_KEYVAULT_NAME'] = form["AZURE_KEYVAULT_NAME"] - os.environ['ENV'] = ENV + os.environ['ENV'] = app.config['ENV'] os.environ['INSECURE_OAUTH'] = form["INSECURE_OAUTH"] os.environ['JENKINS_ENABLED'] = form["JENKINS_ENABLED"] os.environ['JENKINS_HOST'] = form["JENKINS_HOST"] @@ -286,7 +278,43 @@ def set_env_variables(form): os.environ['SNOW_INSTANCE_NAME'] = form["SNOW_INSTANCE_NAME"] os.environ['SNOW_PASSWORD'] = form["SNOW_PASSWORD"] os.environ['SNOW_USERNAME'] = form["SNOW_USERNAME"] - os.environ['VERSION'] = VERSION + os.environ['VERSION'] = app.config['VERSION'] + app.config['APP_EXT_URL'] = form["APP_EXT_URL"] + app.config['AUTH_TYPE'] = form["AUTH_TYPE"] + app.config['AZAD_AUTHORITY'] = form["AZAD_AUTHORITY"] + app.config['AZAD_CLIENT_ID'] = form["AZAD_CLIENT_ID"] + app.config['AZAD_CLIENT_SECRET'] = form["AZAD_CLIENT_SECRET"] + app.config['AZURE_KEYVAULT_NAME'] = form["AZURE_KEYVAULT_NAME"] + app.config['ENV'] = app.config['ENV'] + app.config['INSECURE_OAUTH'] = form["INSECURE_OAUTH"] + app.config['JENKINS_ENABLED'] = form["JENKINS_ENABLED"] + app.config['JENKINS_HOST'] = form["JENKINS_HOST"] + app.config['JENKINS_KEY'] = form["JENKINS_KEY"] + app.config['JENKINS_PROJECT'] = form["JENKINS_PROJECT"] + app.config['JENKINS_STAGING_PROJECT'] = form["JENKINS_STAGING_PROJECT"] + app.config['JENKINS_TOKEN'] = form["JENKINS_TOKEN"] + app.config['JENKINS_USER'] = form["JENKINS_USER"] + app.config['LDAP_BASE_DN'] = form["LDAP_BASE_DN"] + app.config['LDAP_BIND_USER_DN'] = form["LDAP_BIND_USER_DN"] + app.config['LDAP_BIND_USER_PASSWORD'] = form["LDAP_BIND_USER_PASSWORD"] + app.config['LDAP_GROUP_DN'] = form["LDAP_GROUP_DN"] + app.config['LDAP_HOST'] = form["LDAP_HOST"] + app.config['LDAP_PORT'] = form["LDAP_PORT"] + app.config['LDAP_USER_DN'] = form["LDAP_USER_DN"] + app.config['LDAP_USER_LOGIN_ATTR'] = form["LDAP_USER_LOGIN_ATTR"] + app.config['LDAP_USER_RDN_ATTR'] = form["LDAP_USER_RDN_ATTR"] + app.config['PROD_DB_URI'] = form["PROD_DB_URI"] + app.config['SMTP_ADMIN_EMAIL'] = form["SMTP_ADMIN_EMAIL"] + app.config['SMTP_HOST'] = form["SMTP_HOST"] + app.config['SMTP_PASSWORD'] = form["SMTP_PASSWORD"] + app.config['SMTP_USER'] = form["SMTP_USER"] + app.config['SNOW_ENABLED'] = form["SNOW_ENABLED"] + app.config['SNOW_CLIENT_ID'] = form["SNOW_CLIENT_ID"] + app.config['SNOW_CLIENT_SECRET'] = form["SNOW_CLIENT_SECRET"] + app.config['SNOW_INSTANCE_NAME'] = form["SNOW_INSTANCE_NAME"] + app.config['SNOW_PASSWORD'] = form["SNOW_PASSWORD"] + app.config['SNOW_USERNAME'] = form["SNOW_USERNAME"] + app.config['VERSION'] = app.config['VERSION'] @admin.route('/dangerous/delete_all', methods=['POST']) @@ -300,7 +328,7 @@ def delete_all_data(): nav_subsubcat='', nav_curpage={"name": "Unauthorized"}) try: - if ENV == 'test': + if app.config['ENV'] == 'test': # Ensure all connections to the database are closed db.session.close() db.engine.dispose() diff --git a/src/vr/admin/routes/users.py b/src/vr/admin/routes/users.py index 5e461b33..95f12ae3 100644 --- a/src/vr/admin/routes/users.py +++ b/src/vr/admin/routes/users.py @@ -8,7 +8,6 @@ from sqlalchemy import text from flask import request, render_template, session, redirect, url_for, json from flask_login import login_required -from config_engine import AUTH_TYPE from vr.functions.table_functions import load_table, update_table from vr.admin.email_alerts import send_registration_email from vr.assets.model.businessapplications import BusinessApplications @@ -296,7 +295,7 @@ def add_new_user(): first_name=firstname, last_name=lastname, is_active=False, - auth_type=AUTH_TYPE, + auth_type=app.config['AUTH_TYPE'], otp_secret=otp_secret, user_type='system', avatar_path='/static/images/default_profile_avatar.jpg' diff --git a/src/vr/api/integrations/servicenow.py b/src/vr/api/integrations/servicenow.py index 5f76555a..ec06d99f 100644 --- a/src/vr/api/integrations/servicenow.py +++ b/src/vr/api/integrations/servicenow.py @@ -1,13 +1,12 @@ from flask import jsonify, request import requests import json +from vr import app from vr.api import api from vr.admin.oauth2 import require_oauth from authlib.integrations.flask_oauth2 import current_token from vr.admin.auth_functions import verify_api_key, get_token_auth_header from vr.functions.routing_functions import check_entity_permissions -from vr.assets.model.applicationprofiles import ApplicationProfiles, ApplicationProfilesSchema -from config_engine import SNOW_PASSWORD, SNOW_CLIENT_SECRET, SNOW_INSTANCE_NAME, SNOW_CLIENT_ID, SNOW_USERNAME ERROR_RESP = "Error: Invalid API Request" @@ -184,11 +183,11 @@ def create_new_collaboration_tools(snow_obj, app_name, app_desc): return sys_id_map # ServiceNow instance details -INSTANCE_NAME = SNOW_INSTANCE_NAME -CLIENT_ID = SNOW_CLIENT_ID -CLIENT_SECRET = SNOW_CLIENT_SECRET -USERNAME = SNOW_USERNAME -PASSWORD = SNOW_PASSWORD +INSTANCE_NAME = app.config['SNOW_INSTANCE_NAME'] +CLIENT_ID = app.config['SNOW_CLIENT_ID'] +CLIENT_SECRET = app.config['SNOW_CLIENT_SECRET'] +USERNAME = app.config['SNOW_USERNAME'] +PASSWORD = app.config['SNOW_PASSWORD'] TOKEN_URL = f'https://{INSTANCE_NAME}.service-now.com/oauth_token.do' TOKEN_DATA = { diff --git a/src/vr/api/vulns/jenkins_webhook.py b/src/vr/api/vulns/jenkins_webhook.py index 28f1824e..0b2e8b2d 100644 --- a/src/vr/api/vulns/jenkins_webhook.py +++ b/src/vr/api/vulns/jenkins_webhook.py @@ -2,10 +2,11 @@ import datetime from threading import Thread from flask import jsonify, request, json -from vr import db +from vr import db, app from vr.api import api from requests.auth import HTTPBasicAuth -from config_engine import JENKINS_USER, JENKINS_KEY, JENKINS_PROJECT, JENKINS_HOST, JENKINS_TOKEN +# from config_engine import JENKINS_USER, JENKINS_KEY, JENKINS_PROJECT, JENKINS_HOST, JENKINS_TOKEN +from config_engine import getConfigs from vr.admin.oauth2 import require_oauth from sqlalchemy import text from vr.assets.model.cicdpipelinebuilds import CICDPipelineBuilds @@ -25,6 +26,7 @@ @api.route('/api/jenkins_webhook', methods=['POST']) @require_oauth('write:vulnerabilities') def jenkins_webhook(): + getConfigs(app.config) all = request.form payload_dict = json.loads(all['payload']) ref = payload_dict['ref'] diff --git a/src/vr/api/vulns/vulnerabilities.py b/src/vr/api/vulns/vulnerabilities.py index 68360c8b..b151f5bf 100644 --- a/src/vr/api/vulns/vulnerabilities.py +++ b/src/vr/api/vulns/vulnerabilities.py @@ -16,7 +16,6 @@ from vr.admin.oauth2 import require_oauth from vr.admin.functions import db_connection_handler from authlib.integrations.flask_oauth2 import current_token -from config_engine import ENV import re @@ -295,7 +294,7 @@ def _add_new_vulns(new_vulns, engine): def _setup_duplicate_vulns(source_type, dup_vulns): sourced_dup_vulns = [] for vuln in dup_vulns: - if ENV == 'test': + if app.config['ENV'] == 'test': vuln['LastModifiedDate'] = datetime.datetime.utcnow().replace(microsecond=0) if vuln['ReleaseDate']: vuln['ReleaseDate'] = datetime.datetime.strptime(vuln['ReleaseDate'], '%Y-%m-%d %H:%M:%S') diff --git a/src/vr/db_models/setup.py b/src/vr/db_models/setup.py index b36fdc22..cb37b344 100644 --- a/src/vr/db_models/setup.py +++ b/src/vr/db_models/setup.py @@ -1,11 +1,6 @@ from datetime import datetime from sqlalchemy.types import TEXT, DECIMAL from flask import jsonify -from config_engine import ENV -if ENV == 'test': - from sqlalchemy.dialects.sqlite import TEXT as LONGTEXT -else: - from sqlalchemy.dialects.mysql import LONGTEXT from flask_sqlalchemy import SQLAlchemy from sqlalchemy.dialects import mysql from flask_login import UserMixin @@ -176,9 +171,9 @@ class TmControls(db.Model): __tablename__ = 'TmControls' ID = db.Column(db.Integer, primary_key=True) AddDate = db.Column(db.DateTime, index=True, default=datetime.utcnow, nullable=False) - Control = db.Column(LONGTEXT) + Control = db.Column(db.String(20000)) Type = db.Column(db.String(8)) - Description = db.Column(LONGTEXT) + Description = db.Column(db.String(20000)) Lambda = db.Column(db.String(1)) Process = db.Column(db.String(1)) Server = db.Column(db.String(1)) diff --git a/src/vr/db_models/setup_2.py b/src/vr/db_models/setup_2.py index d47ba210..0d98c122 100644 --- a/src/vr/db_models/setup_2.py +++ b/src/vr/db_models/setup_2.py @@ -2,8 +2,8 @@ from sqlalchemy.types import TEXT, DECIMAL from sqlalchemy import MetaData from vr.admin.functions import db_connection_handler -from config_engine import ENV -if ENV == 'test': +from vr import app +if app.config['ENV'] == 'test': from sqlalchemy.dialects.sqlite import TEXT as LONGTEXT else: from sqlalchemy.dialects.mysql import LONGTEXT diff --git a/src/vr/orchestration/web/pipeline_jobs.py b/src/vr/orchestration/web/pipeline_jobs.py index e9fcc7ee..2ef5390b 100644 --- a/src/vr/orchestration/web/pipeline_jobs.py +++ b/src/vr/orchestration/web/pipeline_jobs.py @@ -11,7 +11,6 @@ from vr.orchestration.model.cicdpipelines import CICDPipelines, CICDPipelinesSchema from vr.orchestration.model.pipelinejobs import PipelineJobs, PipelineJobsSchema from vr.orchestration.web.pipeline_stage_data import OPTS -from config_engine import JENKINS_USER, JENKINS_KEY, JENKINS_STAGING_PROJECT, JENKINS_HOST, JENKINS_TOKEN NAV = { @@ -248,7 +247,7 @@ def validate_cicd_pipeline_stage(appid): "Content-Type": "application/x-www-form-urlencoded" } data = { - 'token': JENKINS_TOKEN, + 'token': app.config['JENKINS_TOKEN'], 'GIT_URL': git_url, 'GIT_BRANCH': git_branch, 'APP_NAME': app_name, @@ -261,8 +260,8 @@ def validate_cicd_pipeline_stage(appid): 'TARGET_URL': target_url } - url = f'{JENKINS_HOST}/job/{JENKINS_STAGING_PROJECT}/buildWithParameters' - resp = requests.post(url, headers=headers, data=data, auth=HTTPBasicAuth(JENKINS_USER, JENKINS_KEY)) + url = f"{app.config['JENKINS_HOST']}/job/{app.config['JENKINS_STAGING_PROJECT']}/buildWithParameters" + resp = requests.post(url, headers=headers, data=data, auth=HTTPBasicAuth(app.config['JENKINS_USER'], app.config['JENKINS_KEY'])) return str(200) diff --git a/src/vr/templates/admin/settings.html b/src/vr/templates/admin/settings.html index a7564e1a..f5da3318 100644 --- a/src/vr/templates/admin/settings.html +++ b/src/vr/templates/admin/settings.html @@ -134,7 +134,7 @@

General Settings

{% elif key == "PROD_DB_URI" %} -
+
{% elif key == "ENV" or key == "VERSION" %} {% else %} @@ -176,7 +176,7 @@

SMTP Settings

{% if key == 'SMTP_PASSWORD' %} -
+
{% else %} {% endif %} @@ -199,10 +199,10 @@

Jenkins Settings

{% elif key == 'JENKINS_KEY' %} -
+
{% elif key == 'JENKINS_TOKEN' %} -
+
{% else %} @@ -229,10 +229,10 @@

ServiceNOW Settings

{% elif key == 'SNOW_CLIENT_SECRET' %} -
+
{% elif key == 'SNOW_PASSWORD' %} -
+
{% else %} diff --git a/src/vr/vulns/web/findings.py b/src/vr/vulns/web/findings.py index 8c9938c8..06d44322 100644 --- a/src/vr/vulns/web/findings.py +++ b/src/vr/vulns/web/findings.py @@ -25,7 +25,6 @@ import base64 from io import StringIO from flask import Response -from config_engine import ENV from vr.functions.ml_functions import predict_vuln_validity from vr.vulns.model.cvssbasescoresv3 import CVSSBaseScoresV3 from vr.vulns.model.cvssbasescoresv3extensions import CVSSBaseScoresV3Extensions @@ -42,7 +41,7 @@ UNAUTH_STATUS = "403.html" SERVER_ERR_STATUS = "500.html" VULN_STATUS_IS_NOT_CLOSED = "Vulnerabilities.Status NOT LIKE 'Closed-%' AND Vulnerabilities.Status NOT LIKE 'Open-RiskAccepted-%'" -test = ENV +test = app.config['ENV'] if test == 'test': ISO_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" else: diff --git a/src/vr/vulns/web/testing.py b/src/vr/vulns/web/testing.py index 32e604b8..c69f7122 100644 --- a/src/vr/vulns/web/testing.py +++ b/src/vr/vulns/web/testing.py @@ -11,8 +11,8 @@ from vr.vulns.model.vulnerabilityscans import VulnerabilityScans, VulnerabilityScansSchema from vr.functions.table_functions import load_table, update_table from requests.auth import HTTPBasicAuth -from config_engine import JENKINS_USER, JENKINS_KEY, JENKINS_PROJECT, JENKINS_HOST, JENKINS_TOKEN from vr.assets.model.applicationprofiles import ApplicationProfiles, ApplicationProfilesSchema +from vr import app NAV = { @@ -115,14 +115,14 @@ def on_demand_testing(): "Content-Type": "application/x-www-form-urlencoded" } data = { - 'token': JENKINS_TOKEN, + 'token': app.config['JENKINS_TOKEN'], 'GIT_URL': git_url, 'TESTS': tests_to_run.upper(), 'GIT_BRANCH': git_branch, 'APP_NAME': app_name } - url = f'{JENKINS_HOST}/job/{JENKINS_PROJECT}/buildWithParameters' - resp = requests.post(url, headers=headers, data=data, auth=HTTPBasicAuth(JENKINS_USER, JENKINS_KEY)) + url = f"{app.config['JENKINS_HOST']}/job/{app.config['JENKINS_PROJECT']}/buildWithParameters" + resp = requests.post(url, headers=headers, data=data, auth=HTTPBasicAuth(app.config['JENKINS_USER'], app.config['JENKINS_KEY'])) return redirect(request.referrer) From 9edb485f6d54f026937b347d8069aa844047bf7f Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 22 Mar 2024 08:04:28 -0700 Subject: [PATCH 49/51] Feature/jenkins updates (#519) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py * Update updates.py * Update updates.py * Update settings.py * update to settings update without restart * Update run.py * Update pipeline-config.yaml --- pipeline-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-config.yaml b/pipeline-config.yaml index 2a9eb073..d19730b0 100644 --- a/pipeline-config.yaml +++ b/pipeline-config.yaml @@ -9,7 +9,7 @@ stages: branches: - release unitTesting: - enabled: false + enabled: true branches: [] secretScanning: enabled: false From 76926b35975dcf8330a2d17ec2bc87fec6fa4605 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:37:27 -0700 Subject: [PATCH 50/51] Feature/jenkins updates (#521) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py * Update updates.py * Update updates.py * Update settings.py * update to settings update without restart * Update run.py * Update pipeline-config.yaml * Update __init__.py --- src/vr/__init__.py | 85 ++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/src/vr/__init__.py b/src/vr/__init__.py index 2f3ccf73..90fe1b09 100644 --- a/src/vr/__init__.py +++ b/src/vr/__init__.py @@ -214,50 +214,47 @@ def connect_to_db(): return cur, db def getPersistentConfig(): - try: - cur, db = connect_to_db() - sql = 'SELECT * FROM AppConfig WHERE 1=1' - cur.execute(sql) - row = cur.fetchone() - if row[2]: - app.config['APP_EXT_URL'] = row[3] - app.config['AUTH_TYPE'] = row[4] - app.config['AZAD_AUTHORITY'] = row[5] - app.config['AZAD_CLIENT_ID'] = row[6] - app.config['AZAD_CLIENT_SECRET'] = row[7] - app.config['AZURE_KEYVAULT_NAME'] = row[8] - app.config['ENV'] = row[9] - app.config['INSECURE_OAUTH'] = row[10] - app.config['JENKINS_ENABLED'] = row[37] - app.config['JENKINS_HOST'] = row[11] - app.config['JENKINS_KEY'] = row[12] - app.config['JENKINS_PROJECT'] = row[13] - app.config['JENKINS_STAGING_PROJECT'] = row[14] - app.config['JENKINS_TOKEN'] = row[15] - app.config['JENKINS_USER'] = row[16] - app.config['LDAP_BASE_DN'] = row[17] - app.config['LDAP_BIND_USER_DN'] = row[18] - app.config['LDAP_BIND_USER_PASSWORD'] = row[19] - app.config['LDAP_GROUP_DN'] = row[20] - app.config['LDAP_HOST'] = row[21] - app.config['LDAP_PORT'] = row[22] - app.config['LDAP_USER_DN'] = row[23] - app.config['LDAP_USER_LOGIN_ATTR'] = row[24] - app.config['LDAP_USER_RDN_ATTR'] = row[25] - app.config['PROD_DB_URI'] = row[26] - app.config['SMTP_ADMIN_EMAIL'] = row[27] - app.config['SMTP_HOST'] = row[28] - app.config['SMTP_PASSWORD'] = row[29] - app.config['SMTP_USER'] = row[30] - app.config['SNOW_ENABLED'] = row[38] - app.config['SNOW_CLIENT_ID'] = row[31] - app.config['SNOW_CLIENT_SECRET'] = row[32] - app.config['SNOW_INSTANCE_NAME'] = row[33] - app.config['SNOW_PASSWORD'] = row[34] - app.config['SNOW_USERNAME'] = row[35] - app.config['VERSION'] = row[36] - except: - print('AppConfig Database table is either unreachable or not setup.') + cur, db = connect_to_db() + sql = 'SELECT * FROM AppConfig WHERE 1=1' + cur.execute(sql) + row = cur.fetchone() + if row[2]: + app.config['APP_EXT_URL'] = row[3] + app.config['AUTH_TYPE'] = row[4] + app.config['AZAD_AUTHORITY'] = row[5] + app.config['AZAD_CLIENT_ID'] = row[6] + app.config['AZAD_CLIENT_SECRET'] = row[7] + app.config['AZURE_KEYVAULT_NAME'] = row[8] + app.config['ENV'] = row[9] + app.config['INSECURE_OAUTH'] = row[10] + app.config['JENKINS_ENABLED'] = row[37] + app.config['JENKINS_HOST'] = row[11] + app.config['JENKINS_KEY'] = row[12] + app.config['JENKINS_PROJECT'] = row[13] + app.config['JENKINS_STAGING_PROJECT'] = row[14] + app.config['JENKINS_TOKEN'] = row[15] + app.config['JENKINS_USER'] = row[16] + app.config['LDAP_BASE_DN'] = row[17] + app.config['LDAP_BIND_USER_DN'] = row[18] + app.config['LDAP_BIND_USER_PASSWORD'] = row[19] + app.config['LDAP_GROUP_DN'] = row[20] + app.config['LDAP_HOST'] = row[21] + app.config['LDAP_PORT'] = row[22] + app.config['LDAP_USER_DN'] = row[23] + app.config['LDAP_USER_LOGIN_ATTR'] = row[24] + app.config['LDAP_USER_RDN_ATTR'] = row[25] + app.config['PROD_DB_URI'] = row[26] + app.config['SMTP_ADMIN_EMAIL'] = row[27] + app.config['SMTP_HOST'] = row[28] + app.config['SMTP_PASSWORD'] = row[29] + app.config['SMTP_USER'] = row[30] + app.config['SNOW_ENABLED'] = row[38] + app.config['SNOW_CLIENT_ID'] = row[31] + app.config['SNOW_CLIENT_SECRET'] = row[32] + app.config['SNOW_INSTANCE_NAME'] = row[33] + app.config['SNOW_PASSWORD'] = row[34] + app.config['SNOW_USERNAME'] = row[35] + app.config['VERSION'] = row[36] getPersistentConfig() From 7dd90fa11418004ade7ff847c0665f491aa17906 Mon Sep 17 00:00:00 2001 From: bkaiserinfosec <49665796+bkaiserinfosec@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:19:14 -0700 Subject: [PATCH 51/51] Feature/jenkins updates (#523) * Update security_quality_gate.py * Update Jenkinsfile * Update security_quality_gate.py * update settings and groups * add function for table updates * updated function for db updates * Update updates.py * Update updates.py * Update updates.py * Update settings.py * update to settings update without restart * Update run.py * Update pipeline-config.yaml * Update __init__.py * Update __init__.py --- src/vr/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vr/__init__.py b/src/vr/__init__.py index 90fe1b09..e184511e 100644 --- a/src/vr/__init__.py +++ b/src/vr/__init__.py @@ -218,7 +218,7 @@ def getPersistentConfig(): sql = 'SELECT * FROM AppConfig WHERE 1=1' cur.execute(sql) row = cur.fetchone() - if row[2]: + if row and row[2]: app.config['APP_EXT_URL'] = row[3] app.config['AUTH_TYPE'] = row[4] app.config['AZAD_AUTHORITY'] = row[5]