From cb429b4b98503bbf6f1cdb1fdc48be7d641926e8 Mon Sep 17 00:00:00 2001 From: SOOS-JAlvarez Date: Fri, 3 Nov 2023 10:30:01 -0300 Subject: [PATCH 01/26] PA-11335 Rewrite of Dast wrapper on typescript/auth cleanup --- .gitignore | 230 +-- .prettierrc | 6 + .vscode/settings.json | 19 + Dockerfile | 72 +- config-example.yml | 15 - helpers/__init__.py | 0 helpers/auth.py | 363 ----- helpers/constants.py | 78 - helpers/utils.py | 246 ---- main.py | 1258 ----------------- package-lock.json | 221 +++ package.json | 43 + scripts/blindxss.js | 74 - scripts/httpsender/request_cookies.js | 24 - src/env.d.ts | 7 + src/hooks/helpers/auth.py | 382 +++++ {helpers => src/hooks/helpers}/blindxss.py | 7 +- .../hooks/helpers}/browserstorage.py | 0 .../hooks/helpers}/configuration.py | 12 +- src/hooks/helpers/constants.py | 33 + .../hooks/helpers}/custom_cookies.py | 6 +- .../hooks/helpers}/custom_headers.py | 4 +- src/hooks/helpers/globals.py | 6 + {helpers => src/hooks/helpers}/log.py | 25 +- .../hooks/helpers}/requestheaders.py | 0 src/hooks/helpers/utils.py | 71 + {model => src/hooks/model}/log_level.py | 2 +- .../hooks/model}/target_availability_check.py | 0 .../hooks/requirements.txt | 5 +- .../hooks/soos_zap_hook.py | 26 +- src/index.ts | 544 +++++++ .../traditional-json-headers/report.json | 0 .../traditional-json-headers/template.yaml | 0 .../reports}/traditional-json/report.json | 0 .../reports}/traditional-json/template.yaml | 0 src/utils/Logger.ts | 94 ++ src/utils/ZapCommandGenerator.ts | 99 ++ src/utils/constants.ts | 30 + src/utils/enums.ts | 47 + src/utils/utilities.ts | 109 ++ tests/tests.py | 42 - tsconfig.json | 19 + 42 files changed, 1822 insertions(+), 2397 deletions(-) create mode 100644 .prettierrc create mode 100644 .vscode/settings.json delete mode 100644 config-example.yml delete mode 100644 helpers/__init__.py delete mode 100644 helpers/auth.py delete mode 100644 helpers/constants.py delete mode 100644 helpers/utils.py delete mode 100644 main.py create mode 100644 package-lock.json create mode 100644 package.json delete mode 100644 scripts/blindxss.js delete mode 100644 scripts/httpsender/request_cookies.js create mode 100644 src/env.d.ts create mode 100644 src/hooks/helpers/auth.py rename {helpers => src/hooks/helpers}/blindxss.py (91%) rename {helpers => src/hooks/helpers}/browserstorage.py (100%) rename {helpers => src/hooks/helpers}/configuration.py (92%) create mode 100644 src/hooks/helpers/constants.py rename {helpers => src/hooks/helpers}/custom_cookies.py (81%) rename {helpers => src/hooks/helpers}/custom_headers.py (84%) create mode 100644 src/hooks/helpers/globals.py rename {helpers => src/hooks/helpers}/log.py (72%) rename {helpers => src/hooks/helpers}/requestheaders.py (100%) create mode 100644 src/hooks/helpers/utils.py rename {model => src/hooks/model}/log_level.py (90%) rename {model => src/hooks/model}/target_availability_check.py (100%) rename requirements.txt => src/hooks/requirements.txt (56%) rename hooks/soos_dast_hook.py => src/hooks/soos_zap_hook.py (74%) create mode 100644 src/index.ts rename {reports => src/reports}/traditional-json-headers/report.json (100%) rename {reports => src/reports}/traditional-json-headers/template.yaml (100%) rename {reports => src/reports}/traditional-json/report.json (100%) rename {reports => src/reports}/traditional-json/template.yaml (100%) create mode 100644 src/utils/Logger.ts create mode 100644 src/utils/ZapCommandGenerator.ts create mode 100644 src/utils/constants.ts create mode 100644 src/utils/enums.ts create mode 100644 src/utils/utilities.ts delete mode 100644 tests/tests.py create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore index 5318f5b..8225baa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,228 +1,2 @@ -.DS_Store - -### Python template -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -### JetBrains template -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 - -.idea - -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - -### VisualStudioCode template -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -# Local History for Visual Studio Code -.history/ - +/node_modules +/dist diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..1fc490c --- /dev/null +++ b/.prettierrc @@ -0,0 +1,6 @@ +{ + "parser": "typescript", + "printWidth": 100, + "quoteProps": "consistent", + "endOfLine": "auto" +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9db20b3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,19 @@ +{ + "editor.tabSize": 2, + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "prettier.configPath": "./.prettierrc", + "cSpell.words": [ + "apiscan", + "DAST", + "fullscan", + "httpsender", + "Levelkey", + "nargs", + "SARIF", + "SOOS", + "SOOSDAST", + "SPIDERED" + ], + "sarif-viewer.connectToGithubCodeScanning": "off", + } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 769b070..f54baa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,39 @@ -# if the image or tag changes, make sure to update the scan structure tool name and version FROM soosio/zap2docker-soos as base USER root -COPY ./main.py ./requirements.txt ./VERSION.txt ./ -COPY ./helpers helpers/ -COPY ./hooks hooks/ -COPY ./model model/ -COPY ./scripts/httpsender /home/zap/.ZAP/scripts/scripts/httpsender/ -RUN chmod 777 /home/zap/.ZAP/scripts/scripts/httpsender/ +# Install nodejs version based on NODE_MAJOR +ENV NODE_MAJOR 18 +RUN apt-get update +RUN apt-get install -y ca-certificates curl gnupg +RUN mkdir -p /etc/apt/keyrings +RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +RUN apt-get update +RUN apt-get install -y nodejs -COPY ./reports/traditional-json /zap/reports/traditional-json -COPY ./reports/traditional-json-headers /zap/reports/traditional-json-headers -RUN chmod -R 444 /zap/reports/traditional-json -RUN chmod -R 444 /zap/reports/traditional-json-headers +COPY ./src/ ./src/ +COPY ./tsconfig.json ./ +COPY ./package.json ./ -RUN pip3 install -r requirements.txt && mkdir /zap/wrk && cd /opt \ - && wget -qO- -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz \ - && tar -xvzf geckodriver.tar.gz \ - && chmod +x geckodriver \ - && ln -s /opt/geckodriver /usr/bin/geckodriver \ - && export PATH=$PATH:/usr/bin/geckodriver +RUN pip3 install -r ./src/hooks/requirements.txt -# Set up the Chrome PPA -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - -RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list +RUN mkdir /zap/wrk && cd /opt \ + && wget -qO- -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz \ + && tar -xvzf geckodriver.tar.gz \ + && chmod +x geckodriver \ + && ln -s /opt/geckodriver /usr/bin/geckodriver \ + && export PATH=$PATH:/usr/bin/geckodriver + +RUN cd /zap/plugin && \ + rm -rf ascanrules-* && wget https://github.com/zaproxy/zap-extensions/releases/download/ascanrules-v49/ascanrules-release-49.zap && \ + rm -rf ascanrulesBeta-* && wget https://github.com/zaproxy/zap-extensions/releases/download/ascanrulesBeta-v44/ascanrulesBeta-beta-44.zap && \ + rm -rf commonlib-* && wget https://github.com/zaproxy/zap-extensions/releases/download/commonlib-v1.12.0/commonlib-release-1.12.0.zap && \ + rm -rf network-* && wget https://github.com/zaproxy/zap-extensions/releases/download/network-v0.6.0/network-beta-0.6.0.zap && \ + rm -rf oast-* && wget https://github.com/zaproxy/zap-extensions/releases/download/oast-v0.14.0/oast-beta-0.14.0.zap && \ + rm -rf pscanrules-* && wget https://github.com/zaproxy/zap-extensions/releases/download/pscanrules-v44/pscanrules-release-44.zap && \ + rm -rf pscanrulesBeta-* && wget https://github.com/zaproxy/zap-extensions/releases/download/pscanrulesBeta-v31/pscanrulesBeta-beta-31.zap && \ + chown -R zap:zap /zap # Set up Chrome version to be used ARG CHROME_VERSION="114.0.5735.133-1" @@ -47,22 +56,13 @@ RUN unzip $CHROMEDRIVER_DIR/chromedriver* -d $CHROMEDRIVER_DIR # Put Chromedriver into the PATH ENV PATH $CHROMEDRIVER_DIR:$PATH -RUN cd /zap/plugin && \ - rm -rf ascanrules-* && wget https://github.com/zaproxy/zap-extensions/releases/download/ascanrules-v49/ascanrules-release-49.zap && \ - rm -rf ascanrulesBeta-* && wget https://github.com/zaproxy/zap-extensions/releases/download/ascanrulesBeta-v44/ascanrulesBeta-beta-44.zap && \ - rm -rf commonlib-* && wget https://github.com/zaproxy/zap-extensions/releases/download/commonlib-v1.12.0/commonlib-release-1.12.0.zap && \ - rm -rf network-* && wget https://github.com/zaproxy/zap-extensions/releases/download/network-v0.6.0/network-beta-0.6.0.zap && \ - rm -rf oast-* && wget https://github.com/zaproxy/zap-extensions/releases/download/oast-v0.14.0/oast-beta-0.14.0.zap && \ - rm -rf pscanrules-* && wget https://github.com/zaproxy/zap-extensions/releases/download/pscanrules-v44/pscanrules-release-44.zap && \ - rm -rf pscanrulesBeta-* && wget https://github.com/zaproxy/zap-extensions/releases/download/pscanrulesBeta-v31/pscanrulesBeta-beta-31.zap && \ - chown -R zap:zap /zap - - -FROM base as test -COPY ./tests tests/ +COPY ./src/reports/traditional-json /zap/reports/traditional-json +COPY ./src/reports/traditional-json-headers /zap/reports/traditional-json-headers +RUN chmod -R 444 /zap/reports/traditional-json +RUN chmod -R 444 /zap/reports/traditional-json-headers -ENTRYPOINT ["python3", "-m", "unittest", "tests/tests.py"] +RUN npm install -FROM base as production +RUN npm run build -ENTRYPOINT ["python3", "main.py"] +ENTRYPOINT ["node", "dist/index.js"] \ No newline at end of file diff --git a/config-example.yml b/config-example.yml deleted file mode 100644 index a538e37..0000000 --- a/config-example.yml +++ /dev/null @@ -1,15 +0,0 @@ -config: - clientId: SOOS_CLIENT_ID - apiKey: SOOS_API_KEY - projectName: 'SOOS DAST Analysis' - scanMode: 'baseline' - failOnError: 'continue_on_error' - rules: '' - context: - file: '' - user: '' - apiURL: 'https://api.soos.io/api/' - debug: false - ajaxSpider: false - apiScan: - format: 'openapi' diff --git a/helpers/__init__.py b/helpers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/helpers/auth.py b/helpers/auth.py deleted file mode 100644 index e6914ce..0000000 --- a/helpers/auth.py +++ /dev/null @@ -1,363 +0,0 @@ -from os import environ, pathsep -from re import search -from time import sleep -from traceback import print_exc - -from pyotp import TOTP -from requests import post -from selenium import webdriver -from selenium.webdriver.common.action_chains import ActionChains -from selenium.common.exceptions import NoSuchElementException, TimeoutException -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait -from helpers.browserstorage import BrowserStorage -from helpers.utils import array_to_dict, log -from model.log_level import LogLevel - - -class DASTAuth: - driver = None - config = None - - def __init__(self, config=None): - self.config = config - - def setup_context(self, zap, target): - # Set an X-Scanner header so requests can be identified in logs - zap.replacer.add_rule(description='Scanner', enabled=True, matchtype='REQ_HEADER', - matchregex=False, matchstring='X-Scanner', replacement="ZAP") - - context_name = 'ctx-zap-docker' - context_id = zap.context.new_context(context_name) - - import zap_common - zap_common.context_name = context_name - zap_common.context_id = context_id - - # include everything below the target - self.config.auth_include_urls.append(target + '.*') - - # include additional url's - for include in self.config.auth_include_urls: - zap.context.include_in_context(context_name, include) - log(f"Included {include}") - - # exclude all urls that end the authenticated session - if len(self.config.auth_exclude_urls) == 0: - self.config.auth_exclude_urls.append('.*logout.*') - self.config.auth_exclude_urls.append('.*uitloggen.*') - self.config.auth_exclude_urls.append('.*afmelden.*') - self.config.auth_exclude_urls.append('.*signout.*') - - for exclude in self.config.auth_exclude_urls: - zap.context.exclude_from_context(context_name, exclude) - log(f"Excluded {exclude}") - - def setup_webdriver(self): - log('Start webdriver') - - options = webdriver.ChromeOptions() - if not self.config.auth_display: - options.add_argument('--headless') - options.add_argument('--ignore-certificate-errors') - options.add_argument('--no-sandbox') - options.add_argument('--disable-dev-shm-usage') - # NOTE this is needed for the chromedriver path to be found on github actions, where the path is overridden before execution - if environ['CHROMEDRIVER_DIR'] not in environ['PATH']: - log(f"adding {environ['CHROMEDRIVER_DIR']} to path {environ['PATH']}") - environ["PATH"] += pathsep + environ['CHROMEDRIVER_DIR'] - - self.driver = webdriver.Chrome(options=options) - self.driver.set_window_size(1920, 1080) - self.driver.maximize_window() - - def authenticate(self, zap, target): - try: - # setup the zap context - if zap is not None: - self.setup_context(zap, target) - - # perform authentication using selenium - if self.config.auth_login_url: - # setup the webdriver - self.setup_webdriver() - - # login to the application - self.login() - - # find session cookies or tokens and set them in ZAP - self.set_authentication(zap, target) - # perform authentication using a provided Bearer token - elif self.config.auth_bearer_token: - self.add_authorization_header( - zap, f"Bearer {self.config.auth_bearer_token}") - # perform authentication using a simple token endpoint - elif self.config.auth_token_endpoint: - self.login_from_token_endpoint(zap) - # perform authentication to url that grant access_token for oauth - elif self.config.oauth_token_url: - self.login_from_oauth_token_url(zap) - else: - log( - 'No login URL, Token Endpoint or Bearer token provided - skipping authentication', - log_level=LogLevel.WARN - ) - - except Exception: - log(f"error in authenticate: {print_exc()}", log_level=LogLevel.ERROR) - finally: - self.cleanup() - - def set_authentication(self, zap, target): - log('Finding authentication cookies') - # Create an empty session for session cookies - if zap is not None: - zap.httpsessions.add_session_token(target, 'session_token') - zap.httpsessions.create_empty_session(target, 'auth-session') - - # add all found cookies as session cookies - for cookie in self.driver.get_cookies(): - if zap is not None: - zap.httpsessions.set_session_token_value( - target, 'auth-session', cookie['name'], cookie['value']) - log(f"Cookie added: {cookie['name']}={cookie['value']}") - - # add token from cookies if exists - self.add_token_from_cookie(zap, self.driver.get_cookies()) - - # Mark the session as active - if zap is not None: - zap.httpsessions.set_active_session(target, 'auth-session') - log(f"Active session: {zap.httpsessions.active_session(target)}") - - log('Finding authentication headers') - - # try to find JWT tokens in Local Storage and Session Storage and add them as Authorization header - localStorage = BrowserStorage(self.driver, 'localStorage') - sessionStorage = BrowserStorage(self.driver, 'sessionStorage') - self.add_token_from_browser_storage(zap, localStorage) - self.add_token_from_browser_storage(zap, sessionStorage) - - def add_token_from_browser_storage(self, zap, browserStorage): - for key in browserStorage: - log(f"Found key: {key}") - match = search('(eyJ[^"]*)', browserStorage.get(key)) - if match: - auth_header = "Bearer " + match.group() - self.add_authorization_header(zap, auth_header) - - def add_token_from_cookie(self, zap, cookies): - for cookie in cookies: - if cookie['name'] == 'token': - auth_header = "Bearer " + cookie['value'] - self.add_authorization_header(zap, auth_header) - - - def login_from_token_endpoint(self, zap): - log('Fetching authentication token from endpoint') - - response = post(self.config.auth_token_endpoint, data={ - 'username': self.config.auth_username, 'password': self.config.auth_password}) - data = response.json() - auth_header = None - - if "token" in data: - auth_header = f"Bearer {data['token']}" - elif "token_type" in data: - auth_header = f"{data['token_type']} {data['token_type']}" - - if auth_header: - self.add_authorization_header(zap, auth_header) - - def login_from_oauth_token_url(self, zap): - log('Making request to oauth token url') - body = array_to_dict(self.config.oauth_parameters) - response = post(self.config.oauth_token_url, data=body) - data = response.json() - auth_header = None - if "token" in data: - auth_header = f"Bearer {data['token']}" - elif "access_token" in data: - log("setting access_token from oauth response") - auth_header = f"Bearer {data['access_token']}" - - def add_authorization_header(self, zap, auth_token): - if zap is not None: - zap.replacer.add_rule(description='AuthHeader', enabled=True, matchtype='REQ_HEADER', - matchregex=False, matchstring='Authorization', replacement=auth_token) - log(f"Authorization header added: {auth_token}") - - def login(self): - log(f"authenticate using webdriver against URL: {self.config.auth_login_url}") - - self.driver.get(self.config.auth_login_url) - final_submit_button = self.config.auth_submit_field_name - - # wait for the page to load - sleep(5) - - log('automatically finding login elements') - - username_element = None - - # fill out the username field - if self.config.auth_username: - username_element = self.fill_username() - - if self.config.auth_form_type == 'wait_for_password': - log(f"Waiting for {self.config.auth_password_field_name} element to load") - sleep(self.config.auth_delay_time) - - if self.config.auth_form_type == 'multi_page': - continue_button = self.find_element(self.config.auth_submit_field_name, "submit", "//*[@type='submit' or @type='button' or button]" ) - actions = ActionChains(self.driver) - actions.move_to_element(continue_button).click().perform() - final_submit_button = self.config.auth_submit_second_field_name - log(f"Clicked the first submit element for multi page") - sleep(self.config.auth_delay_time) - - # fill out the password field - if self.config.auth_password: - try: - self.fill_password() - except Exception: - log( - 'Did not find the password field - clicking Next button and trying again', log_level=LogLevel.WARN) - - # if the password field was not found, we probably need to submit to go to the password page - # login flow: username -> next -> password -> submit - self.fill_password() - - # fill out the OTP field - if self.config.auth_otp_secret: - try: - self.fill_otp() - except Exception: - log( - 'Did not find the OTP field - clicking Next button and trying again', log_level=LogLevel.WARN) - - # if the OTP field was not found, we probably need to submit to go to the OTP page - # login flow: username -> next -> password -> next -> otp -> submit - self.submit_form(self.config.auth_submit_action, - final_submit_button, self.config.auth_password_field_name) - self.fill_otp() - - # submit - self.submit_form(self.config.auth_submit_action, - final_submit_button, self.config.auth_password_field_name) - - # wait for the page to load - if self.config.auth_check_element: - try: - log('Check element') - WebDriverWait(self.driver, self.config.auth_check_delay).until( - EC.presence_of_element_located((By.XPATH, self.config.auth_check_element))) - except TimeoutException: - log('Check element timeout') - else: - sleep(self.config.auth_check_delay) - - def submit_form(self, submit_action, submit_field_name, password_field_name): - if submit_action == "click": - element = self.find_element( - submit_field_name, "submit", "//*[@type='submit' or @type='button' or button]") - actions = ActionChains(self.driver) - actions.move_to_element(element).click().perform() - log(f"Clicked the {submit_field_name} element") - else: - self.find_element(password_field_name,"password","//input[@type='password' or contains(@name,'ass')]").submit() - log('Submitted the form') - - def fill_username(self): - return self.find_and_fill_element(self.config.auth_username, - self.config.auth_username_field_name, - "input", - "(//input[((@type='text' or @type='email') and contains(@name,'ser')) or (@type='text' or @type='email')])[1]") - - def fill_password(self): - return self.find_and_fill_element(self.config.auth_password, - self.config.auth_password_field_name, - "password", - "//input[@type='password' or contains(@name,'ass')]") - - def fill_otp(self): - totp = TOTP(self.config.auth_otp_secret) - otp = totp.now() - - log(f"Generated OTP: {otp}") - - return self.find_and_fill_element(otp, - self.config.auth_otp_field_name, - "input", - "//input[@type='text' and (contains(@id,'otp') or contains(@name,'otp'))]") - - def find_and_fill_element(self, value, name, element_type, xpath): - element = self.find_element(name, element_type, xpath) - element.clear() - element.send_keys(value) - log(f"Filled the {name} element") - - return element - - # 1. Find by ID attribute (case insensitive) - # 2. Find by Name attribute (case insensitive) - # 3. Find by xpath - # 4. Find by the default xpath if all above fail - def find_element(self, name_or_id_or_xpath, element_type, default_xpath): - element = None - log(f"Trying to find element {name_or_id_or_xpath}") - - if name_or_id_or_xpath: - try: - path = self.build_xpath( - name_or_id_or_xpath, "id", element_type) - element = self.driver.find_element_by_xpath(path) - log(f"Found element {name_or_id_or_xpath} by id") - except NoSuchElementException: - try: - path = self.build_xpath( - name_or_id_or_xpath, "name", element_type) - element = self.driver.find_element_by_xpath(path) - log(f"Found element {name_or_id_or_xpath} by name") - except NoSuchElementException: - try: - element = self.driver.find_element_by_xpath( - name_or_id_or_xpath) - log( - f"Found element {name_or_id_or_xpath} by xpath (name)") - except NoSuchElementException: - try: - element = self.driver.find_element_by_xpath( - default_xpath) - log( - f"Found element {default_xpath} by default xpath") - except NoSuchElementException: - log( - f"Failed to find the element {name_or_id_or_xpath}") - - return element - - def build_xpath(self, name, find_by, element_type): - xpath = "translate(@{0}, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz')='{1}'".format( - find_by, name.lower()) - - if element_type == 'input': - xpath = "//input[({0}) and ({1})]".format(xpath, - "@type='text' or @type='email' or @type='number' or not(@type)") - elif element_type == 'password': - xpath = "//input[({0}) and ({1})]".format(xpath, - "@type='text' or @type='password' or not(@type)") - elif element_type == 'submit': - xpath = "//*[({0}) and ({1})]".format(xpath, - "@type='submit' or @type='button' or button") - else: - xpath = "//*[{0}]".format(xpath) - - log(f"Built xpath: {xpath}") - - return xpath - - def cleanup(self): - if self.driver: - self.driver.quit() diff --git a/helpers/constants.py b/helpers/constants.py deleted file mode 100644 index b4af433..0000000 --- a/helpers/constants.py +++ /dev/null @@ -1,78 +0,0 @@ -DEFAULT_API_URL: str = "https://api.soos.io/api/" -HEADER_SOOS_API_KEY: str = "x-soos-apikey" -HEADER_CONTENT_TYPE: str = "Content-Type" -JSON_HEADER_CONTENT_TYPE: str = "application/json" -MULTIPART_HEADER_CONTENT_TYPE: str = "multipart/form-data" -MAX_RETRY_COUNT: int = 3 -SOOS_CLIENT_ID_KEY: str = "SOOS_CLIENT_ID" -SOOS_API_KEY: str = "SOOS_API_KEY" -DEFAULT_INTEGRATION_NAME: str = "None" -DEFAULT_INTEGRATION_TYPE: str = "Script" -DEFAULT_DAST_TOOL: str = "zap" -DEFAULT_DAST_TOOL_VERSION: str = "latest" -SERVER_ERROR_CODES = range(500, 599) -RETRY_DELAY = 3 # seconds -REQUEST_TIMEOUT = 10 # seconds -EMPTY_STRING = '' -FAIL_THE_BUILD = "fail_the_build" -CONTINUE_ON_FAILURE = "continue_on_failure" -AUTH_DELAY_TIME = 5 # seconds - -# SCAN MODES -BASELINE = 'baseline' -FULL_SCAN = 'fullscan' -API_SCAN = 'apiscan' - -# URL PLACEHOLDERS -BASE_URI_PLACEHOLDER = "{soos_base_uri}" -CLIENT_ID_PLACEHOLDER = "{soos_client_id}" -PROJECT_ID_PLACEHOLDER = "{soos_project_id}" -DAST_TOOL_PLACEHOLDER = "{soos_dast_tool}" -ANALYSIS_ID_PLACEHOLDER = "{soos_analysis_id}" - -# FILE PROCESSING -FILE_READ_MODE = "r" -FILE_WRITE_MODE = "x" -UTF_8_ENCODING = "utf-8" - -# OWASP ZAP Constants - for command line options, see https://www.zaproxy.org/docs/docker/full-scan/ -REPORT_SCAN_RESULT_FILENAME = "report.json" -REPORT_SCAN_RESULT_FILE = "/zap/wrk/" + REPORT_SCAN_RESULT_FILENAME -PY_CMD = "python3" -BASE_LINE_SCRIPT = "/zap/zap-baseline.py" -FULL_SCAN_SCRIPT = "/zap/zap-full-scan.py" -API_SCAN_SCRIPT = "/zap/zap-api-scan.py" -CONFIG_FILE_FOLDER = "/zap/config/" -ZAP_TARGET_URL_OPTION = "-t" -ZAP_MINIMUM_LEVEL_OPTION = "-l" -ZAP_RULES_FILE_OPTION = "-c" -ZAP_CONTEXT_FILE_OPTION = "-n" -ZAP_SPIDER_MINUTES_OPTION = "-m" -ZAP_DEBUG_OPTION = "-d" -ZAP_AJAX_SPIDER_OPTION = "-j" -ZAP_FORMAT_OPTION = "-f" -ZAP_JSON_REPORT_OPTION = "-J" -ZAP_OPTIONS = "-z" -ZAP_HOOK_OPTION = "--hook" -# NOTE: ZAP, when performing a 'fullscan', creates a policy called "Default Policy" - it's needed to specify that name in order to change the scan rules. -ZAP_ACTIVE_SCAN_POLICY_NAME = "Default Policy" -URI_START_DAST_ANALYSIS_TEMPLATE = ( - "{soos_base_uri}clients/{soos_client_id}/dast-tools/{soos_dast_tool}/analysis" -) -URI_START_DAST_ANALYSIS_TEMPLATE_v2 = ( - "{soos_base_uri}clients/{soos_client_id}/scan-types/dast/scans" -) -URI_UPLOAD_DAST_RESULTS_TEMPLATE = "{soos_base_uri}clients/{soos_client_id}/projects/{soos_project_id}/dast-tools/{soos_dast_tool}/analysis/{soos_analysis_id}" - -URI_UPLOAD_DAST_RESULTS_TEMPLATE_v2 = "{soos_base_uri}clients/{soos_client_id}/projects/{soos_project_id}/branches/{soos_branch_hash}/scan-types/dast/scans/{soos_analysis_id}" - -URI_PROJECT_DETAILS_TEMPLATE = "{soos_base_uri}projects/{soos_project_id}/details" - -# LOGS -LOG_FORMAT = "%(asctime)s %(message)s" -LOG_DATE_FORMAT = "%m/%d/%Y %I:%M:%S %p %Z" - - -# ZAP SCRIPTS -ZAP_ACTIVE_SCAN_SCRIPTS_FOLDER_PATH = "/home/zap/.ZAP/scripts/scripts/active/" -ZAP_HTTP_SENDER_SCRIPTS_FOLDER_PATH = "/home/zap/.ZAP/scripts/scripts/httpsender/" diff --git a/helpers/utils.py b/helpers/utils.py deleted file mode 100644 index 700079a..0000000 --- a/helpers/utils.py +++ /dev/null @@ -1,246 +0,0 @@ -from base64 import b64encode -from datetime import datetime, timedelta -from html import unescape -from sys import exit -from time import sleep -from typing import Optional, Any, NoReturn, Dict, Iterable -from urllib.parse import unquote - -from requests import Response, get -from requests.exceptions import ( - HTTPError, -) - -import helpers.constants as Constants -from helpers.constants import RETRY_DELAY, REQUEST_TIMEOUT -from model.log_level import LogLevel, loggerFunc -from model.target_availability_check import TargetAvailabilityCheck - -UTF_8: str = 'utf-8' - - -class ErrorAPIResponse: - code: Optional[str] = None - message: Optional[str] = None - - def __init__(self, api_response): - for key in api_response: - self.__setattr__(key, api_response[key]) - - self.code = api_response["code"] if "code" in api_response else None - self.message = api_response["message"] if "message" in api_response else None - - -def log(message: str, log_level: LogLevel = LogLevel.INFO) -> None: - logFunc = loggerFunc.get(log_level) - logFunc(str(message)) - - -def print_line_separator() -> None: - print( - "----------------------------------------------------------------------------------------------------------" - ) - - -def exit_app(e) -> NoReturn: - log(str(e), LogLevel.ERROR) - exit(1) - - -def valid_required(key, value): - if value is None or len(value) == 0: - exit_app(key + " is required") - - -def has_value(prop) -> bool: - return prop is not None and len(prop) > 0 - - -def is_true(prop) -> bool: - return prop is True - - -def check_site_is_available(url: str) -> bool: - log(f"Waiting for {url} to be available") - - check = False - max_time = datetime.utcnow() + timedelta(days=0, minutes=0, seconds=30) - attempt = 1 - - while datetime.utcnow() < max_time: - log(f"Attempt {attempt} to connect to {url}") - try: - check = __send_ping__(url) - - if check is True: - break - - if datetime.utcnow() + timedelta(0, RETRY_DELAY) > max_time: - break - except Exception as error: - pass - - sleep(RETRY_DELAY) - attempt = attempt + 1 - - return check - - -def __send_ping__(target: str) -> bool: - headers = { - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36' - } - response: Response = get( - url=target, - headers=headers, - timeout=REQUEST_TIMEOUT, - verify=False, # nosec - allow_redirects=True, # nosec - ) - - return _check_status(response).is_available() - - -def _check_status(response: Response) -> TargetAvailabilityCheck: - try: - response.raise_for_status() - except HTTPError as error: - log(f"{type(error).__name__}: {response.status_code}") - log(response.text, log_level=LogLevel.DEBUG) - - # 401 status indicates the host is available but may be behind basic auth - if response.status_code == 401: - return TargetAvailabilityCheck(True, response=response) - - return TargetAvailabilityCheck( - False, - response=response, - unavailable_reason=error, - ) - else: - return TargetAvailabilityCheck(True, response=response) - - -def make_call(request) -> Response: - attempt: int = 1 - error_response: Optional[Any] = None - error_message: str = "An error has occurred" - try: - while attempt <= Constants.MAX_RETRY_COUNT: - api_response: Response = request() - - if api_response.ok: - return api_response - else: - error_response = api_response - log( - f"An error has occurred performing the request. Retrying Request: {str(attempt)} attempts" - ) - attempt = attempt + 1 - - if attempt > Constants.MAX_RETRY_COUNT and error_response is not None: - error_response = error_response.json() - error_message = error_response["message"] - - except Exception as error: - log(error) - - exit_app(error_message) - - -def set_generic_value(self, object_key: str, param_key: str, param_value: Optional[Any], is_required=False) -> NoReturn: - if is_required: - valid_required(param_key, param_value) - - if self[object_key]: - self[object_key] = param_value - - -def log_error(api_response: Response) -> None: - log(f"Status Code: {api_response.status_code}", log_level=LogLevel.ERROR) - if api_response.text is not None: - log(f"Response Text: {api_response.text}", log_level=LogLevel.ERROR) - - -def unescape_string(value: str) -> str or None: - if value is None: - return value - - return unescape(unquote(value)) - - -def encode_report(report_json) -> None: - if report_json['site'] is not None: - for site in report_json['site']: - if site['alerts'] is not None: - for alert in site['alerts']: - if alert['instances'] is not None: - for instance in alert['instances']: - instance['base64Uri'] = convert_string_to_b64(instance['uri']) - instance['uri'] = Constants.EMPTY_STRING - - -def convert_string_to_b64(content: str) -> str: - message_bytes = content.encode(UTF_8) - base64_bytes = b64encode(message_bytes) - base64_message = base64_bytes.decode(UTF_8) - return base64_message - - -def process_custom_cookie_header_data(data: str) -> Dict: - values: Dict = dict() - - if data is not None: - dataModified = data.replace('[', Constants.EMPTY_STRING).replace(']', Constants.EMPTY_STRING) - for value in dataModified.split(','): - dict_key, dict_value = value.split(':') - values[dict_key] = dict_value - - return values - - -def read_file(file_path): - with open(file=file_path, mode=Constants.FILE_READ_MODE, encoding=Constants.UTF_8_ENCODING) as file: - return file.read() - - -def write_file(file_path, file_content): - with open(file=file_path, mode=Constants.FILE_WRITE_MODE, encoding=Constants.UTF_8_ENCODING) as file: - file.write(file_content) - file.close() - - -def handle_response(api_response): - if api_response.status_code in range(400, 600): - return ErrorAPIResponse(api_response.json()) - else: - if api_response.reason == "No Content": - return None - else: - return api_response.json() - - -def handle_error(error: ErrorAPIResponse, api: str, attempt: int, max_retry: int): - error_message = f"{api} has an error. Attempt {str(attempt)} of {str(max_retry)}" - raise Exception(f"{error_message}\n{error.code}-{error.message}") - - -def generate_header(api_key: str, content_type: str): - return {'x-soos-apikey': api_key, 'Content-Type': content_type} - - - -def array_to_str(array: Iterable[str]): - - if array is None or sum(1 for element in array) == 0: - return None - - return ' '.join(array) - -def array_to_dict(array: Iterable[str]): - body = [] - for key_value in array: - print(key_value) - key, value = key_value.split(':', 1) - body.append((key, value)) - return body \ No newline at end of file diff --git a/main.py b/main.py deleted file mode 100644 index 0643453..0000000 --- a/main.py +++ /dev/null @@ -1,1258 +0,0 @@ -import base64 -import gzip -import json -import logging -import os -import platform -import sys -from argparse import ArgumentParser, Namespace -from datetime import datetime -import time -from typing import List, Optional, Any, Dict, NoReturn -from collections import OrderedDict - -import requests -import yaml -from requests import Response, put, post, patch - -import helpers.constants as Constants -from helpers.utils import log, valid_required, has_value, exit_app, is_true, print_line_separator, \ - check_site_is_available, log_error, unescape_string, read_file, convert_string_to_b64, generate_header, \ - handle_response, ErrorAPIResponse, array_to_str -from model.log_level import LogLevel - -ANALYSIS_START_TIME = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") -OPERATING_ENVIRONMENT = f'{platform.system()} {platform.release()} {platform.architecture()[0]}' -ANALYSIS_RESULT_POLLING_INTERVAL = 10 # 10 seconds -ANALYSIS_RESULT_MAX_WAIT = 300 # 5 minutes - -with open(os.path.join(os.path.dirname(__file__), "VERSION.txt"), encoding='UTF-8') as version_file: - SCRIPT_VERSION = version_file.read().strip() - -class DASTStartAnalysisResponse: - def __init__(self, dast_analysis_api_response): - self.analysis_id = dast_analysis_api_response[ - "analysisId"] if "analysisId" in dast_analysis_api_response else None - self.branch_hash = dast_analysis_api_response[ - "branchHash"] if "branchHash" in dast_analysis_api_response else None - self.scan_type = dast_analysis_api_response["scanType"] if "scanType" in dast_analysis_api_response else None - self.scan_url = dast_analysis_api_response["scanUrl"] if "scanUrl" in dast_analysis_api_response else None - self.scan_status_url = dast_analysis_api_response[ - "scanStatusUrl"] if "scanStatusUrl" in dast_analysis_api_response else None - self.errors = dast_analysis_api_response["errors"] if "errors" in dast_analysis_api_response else None - self.project_id = dast_analysis_api_response["projectId"] if "projectId" in dast_analysis_api_response else None - if self.project_id is None: - self.project_id = dast_analysis_api_response[ - "projectHash"] if "projectHash" in dast_analysis_api_response else None - - -class SOOSDASTAnalysis: - - def __init__(self): - self.client_id: Optional[str] = None - self.api_key: Optional[str] = None - self.project_name: Optional[str] = None - self.base_uri: Optional[str] = None - self.scan_mode: Optional[str] = None - self.fail_on_error: Optional[str] = None - self.target_url: Optional[str] = None - self.rules_file: Optional[str] = None - self.context_file: Optional[str] = None - self.user_context: Optional[str] = None - self.api_scan_format: Optional[str] = None - self.debug_mode: bool = False - self.app_version: Optional[str] = None - self.ajax_spider_scan: bool = False - self.spider_minutes: Optional[int] = 120 - self.report_request_headers: bool = False - self.on_failure: Optional[str] = None - self.update_addons: bool = False - - # Special Context - loads from script arguments only - self.commit_hash: Optional[str] = None - self.branch_name: Optional[str] = None - self.branch_uri: Optional[str] = None - self.build_version: Optional[str] = None - self.build_uri: Optional[str] = None - self.operating_environment: Optional[str] = None - self.log_level: Optional[str] = None - self.zap_options: Optional[str] = None - self.request_cookies: Optional[str] = None - self.request_header: Optional[str] = None - - # Hardcoded values, used for analysis metadata - self.dast_analysis_tool: str = Constants.DEFAULT_DAST_TOOL - self.dast_analysis_tool_version: str = Constants.DEFAULT_DAST_TOOL_VERSION - self.integration_name: str = Constants.DEFAULT_INTEGRATION_NAME - self.integration_type: str = Constants.DEFAULT_INTEGRATION_TYPE - - # Auth Options - self.auth_auto: Optional[str] = '0' - self.auth_login_url: Optional[str] = None - self.auth_username: Optional[str] = None - self.auth_password: Optional[str] = None - self.auth_username_field_name: Optional[str] = None - self.auth_password_field_name: Optional[str] = None - self.auth_submit_field_name: Optional[str] = None - self.auth_submit_second_field_name: Optional[str] = None - self.auth_submit_action: Optional[str] = None - self.auth_form_type: Optional[str] = None - self.auth_delay_time: Optional[int] = None - self.auth_exclude_urls: Optional[str] = None - self.auth_display: bool = False - self.auth_bearer_token: Optional[str] = None - self.oauth_token_url: Optional[str] = None - self.oauth_parameters: Optional[str] = None - - self.output_format: Optional[str] = None - self.github_pat: Optional[str] = None - self.checkout_dir: Optional[str] = None - self.sarif_destination: Optional[str] = None - self.disable_rules: Optional[str] = None - - # allows passing other command line options directly to the script - self.other_options: Optional[str] = None - - self.scan_mode_map: Dict = { - Constants.BASELINE: self.baseline_scan, - Constants.FULL_SCAN: self.full_scan, - Constants.API_SCAN: self.api_scan - } - - def parse_configuration(self, configuration: Dict, target_url: str): - valid_required("Target URL", target_url) - self.target_url = target_url - self.log_level = configuration.get("level", LogLevel.INFO) - logging.getLogger("SOOS DAST").setLevel(self.log_level) - log(json.dumps(configuration, indent=2), log_level=LogLevel.DEBUG) - for key, value in configuration.items(): - if key == "clientId": - if value is None: - try: - self.client_id = os.environ.get(Constants.SOOS_CLIENT_ID_KEY) - valid_required(key, self.client_id) - except Exception as error: - exit_app(error) - else: - valid_required(key, value) - self.client_id = value - elif key == "apiKey": - if value is None: - try: - self.api_key = os.environ.get(Constants.SOOS_API_KEY) - valid_required(key, self.api_key) - except Exception as error: - exit_app(error) - else: - valid_required(key, value) - self.api_key = value - elif key == "apiURL": - if value is None: - self.base_uri = Constants.DEFAULT_API_URL - else: - self.base_uri = value - elif key == "projectName": - valid_required(key, value) - value = array_to_str(value) - self.project_name = unescape_string(value) - elif key == "scanMode": - valid_required(key, value) - self.scan_mode = value - elif key == "failOnError": - valid_required(key, value) - self.fail_on_error = value - elif key == "rules": - self.rules_file = array_to_str(value) - elif key == "debug": - self.debug_mode = True - elif key == "ajaxSpider": - self.ajax_spider_scan = True - elif key == "contextFile": - self.context_file = array_to_str(value) - elif key == "contextUser": - self.user_context = value - elif key == "fullScanMinutes": - self.spider_minutes = value - elif key == "apiScan": - self.api_scan_format = value["format"] - elif key == "apiScanFormat": - self.api_scan_format = value - elif key == "commitHash": - self.commit_hash = value - elif key == "branchName": - value = array_to_str(value) - self.branch_name = value - elif key == "buildVersion": - self.build_version = value - elif key == "branchURI": - self.branch_uri = value - elif key == "buildURI": - self.build_uri = value - elif key == "operatingEnvironment": - value = array_to_str(value) - self.operating_environment = value - elif key == "integrationName": - if value is None: - self.integration_name = Constants.DEFAULT_INTEGRATION_NAME - else: - value = array_to_str(value) - self.integration_name = value - elif key == "integrationType": - if value is None: - self.integration_type = Constants.DEFAULT_INTEGRATION_TYPE - else: - value = array_to_str(value) - self.integration_type = value - elif key == "appVersion": - value = array_to_str(value) - self.app_version = value - elif key == 'authAuto': - self.auth_auto = '1' - elif key == 'authDisplay': - self.auth_display = value - elif key == 'authUsername': - self.auth_username = value - elif key == 'authPassword': - self.auth_password = value - elif key == 'authLoginURL': - self.auth_login_url = value - elif key == 'authUsernameField': - self.auth_username_field_name = value - elif key == 'authPasswordField': - self.auth_password_field_name = value - elif key == 'authSubmitField': - self.auth_submit_field_name = value - elif key == 'authSecondSubmitField': - self.auth_submit_second_field_name = value - elif key == 'authSubmitAction': - self.auth_submit_action = value - elif key == 'authFormType': - self.auth_form_type = value - elif key == 'authDelayTime': - self.auth_delay_time = value - elif key == "zapOptions": - value = array_to_str(value) - self.zap_options = value - elif key == "requestCookies": - value = array_to_str(value) - self.request_cookies = value - elif key == "requestHeaders": - value = array_to_str(value) - self.request_header = value - elif key == "outputFormat": - self.output_format = value - elif key == "gpat": - self.github_pat = value - elif key =="bearerToken": - self.auth_bearer_token = value - elif key == "reportRequestHeaders": - self.report_request_headers = True if str.lower(value) == "true" else False - elif key == "onFailure": - self.on_failure = value - elif key == "checkoutDir": - self.checkout_dir = value - elif key == "sarifDestination": - self.sarif_destination = value - elif key == "oauthTokenUrl": - self.oauth_token_url = value - elif key == "oauthParameters": - value = array_to_str(value) - self.oauth_parameters = value - elif key == "sarif" and value is not None: - log("Argument 'sarif' is deprecated. Please use --outputFormat='sarif' instead.") - sys.exit(1) - elif key == "updateAddons": - self.update_addons = True if str.lower(value) == "true" else False - elif key == "disableRules": - self.disable_rules = array_to_str(value) - elif key == "otherOptions": - self.other_options = array_to_str(value) - - def __add_target_url_option__(self, args: List[str]) -> NoReturn: - if has_value(self.target_url): - args.append(Constants.ZAP_TARGET_URL_OPTION) - args.append(self.target_url) - else: - exit_app("Target url is required.") - - def __add_rules_file_option__(self, args: List[str]) -> None: - if has_value(self.rules_file): - args.append(Constants.ZAP_RULES_FILE_OPTION) - args.append(self.rules_file) - - def __add_context_file_option__(self, args: List[str]) -> None: - if has_value(self.context_file): - args.append(Constants.ZAP_CONTEXT_FILE_OPTION) - args.append(self.context_file) - - def __add_debug_option__(self, args: List[str]) -> None: - if is_true(self.debug_mode): - args.append(Constants.ZAP_DEBUG_OPTION) - - def __add_ajax_spider_scan_option__(self, args: List[str]) -> None: - if is_true(self.ajax_spider_scan): - args.append(Constants.ZAP_AJAX_SPIDER_OPTION) - - def __add_spider_minutes_option__(self, args: List[str]) -> None: - if has_value(self.spider_minutes): - args.append(Constants.ZAP_SPIDER_MINUTES_OPTION) - args.append(self.spider_minutes) - - def __add_format_option__(self, args: List[str]) -> NoReturn: - if has_value(self.api_scan_format): - args.append(Constants.ZAP_FORMAT_OPTION) - args.append(self.api_scan_format) - elif self.scan_mode == Constants.API_SCAN: - exit_app("Format is required for apiscan mode.") - - def __add_log_level_option__(self, args: List[str]) -> None: - if has_value(self.log_level): - args.append(Constants.ZAP_MINIMUM_LEVEL_OPTION) - args.append(self.log_level) - - def __add_report_file__(self, args: List[str]) -> None: - args.append(Constants.ZAP_JSON_REPORT_OPTION) - args.append(Constants.REPORT_SCAN_RESULT_FILENAME) - - def __add_hook_params__(self) -> None: - log("Adding hook params", LogLevel.DEBUG) - if self.auth_login_url is not None: - os.environ['AUTH_LOGIN_URL'] = self.auth_login_url - if self.auth_username is not None: - os.environ['AUTH_USERNAME'] = self.auth_username - if self.auth_password is not None: - os.environ['AUTH_PASSWORD'] = self.auth_password - if self.request_cookies is not None: - os.environ['CUSTOM_COOKIES'] = self.request_cookies - if self.request_header is not None: - os.environ['CUSTOM_HEADER'] = self.request_header - if self.auth_bearer_token is not None: - os.environ['AUTH_BEARER_TOKEN'] = self.auth_bearer_token - if self.auth_display is not None: - os.environ['AUTH_DISPLAY'] = str(self.auth_display) - if self.auth_submit_field_name is not None: - os.environ['AUTH_SUBMIT_FIELD'] = self.auth_submit_field_name - if self.auth_submit_second_field_name is not None: - os.environ['AUTH_SECOND_SUBMIT_FIELD'] = self.auth_submit_second_field_name - if self.auth_submit_action is not None: - os.environ['AUTH_SUBMIT_ACTION'] = self.auth_submit_action - if self.auth_form_type is not None: - os.environ['AUTH_FORM_TYPE'] = self.auth_form_type - if self.auth_delay_time is not None: - os.environ['AUTH_DELAY_TIME'] = str(self.auth_delay_time) - if self.auth_username_field_name is not None: - os.environ['AUTH_USERNAME_FIELD'] = self.auth_username_field_name - if self.auth_password_field_name is not None: - os.environ['AUTH_PASSWORD_FIELD'] = self.auth_password_field_name - if self.oauth_token_url is not None: - os.environ['OAUTH_TOKEN_URL'] = self.oauth_token_url - if self.oauth_parameters is not None: - os.environ['OAUTH_PARAMETERS'] = self.oauth_parameters - if self.disable_rules is not None: - os.environ['DISABLE_RULES'] = self.disable_rules - - def __add_hook_option__(self, args: List[str]) -> None: - args.append(Constants.ZAP_HOOK_OPTION) - args.append('/zap/hooks/soos_dast_hook.py') - - def __generate_command__(self, args: List[str]) -> str: - self.__add_debug_option__(args) - self.__add_rules_file_option__(args) - self.__add_context_file_option__(args) - self.__add_ajax_spider_scan_option__(args) - self.__add_spider_minutes_option__(args) - - log(f"Auth Login: {str(self.auth_login_url)}") - log(f"Zap Options: {str(self.zap_options)}") - log(f"Cookies: {str(self.request_cookies)}") - log(f"Github PAT: {str(self.github_pat)}") - if (self.scan_mode != Constants.API_SCAN): - self.__add_hook_params__() - self.__add_hook_option__(args) - - self.__add_report_file__(args) - - return " ".join(args) - - def baseline_scan(self) -> str: - args: List[str] = [Constants.PY_CMD, Constants.BASE_LINE_SCRIPT] - - self.__add_target_url_option__(args) - - return self.__generate_command__(args) - - def full_scan(self) -> str: - args: List[str] = [Constants.PY_CMD, Constants.FULL_SCAN_SCRIPT] - - self.__add_target_url_option__(args) - - return self.__generate_command__(args) - - def api_scan(self) -> str: - valid_required("api_scan_format", self.api_scan_format) - args: List[str] = [Constants.PY_CMD, Constants.API_SCAN_SCRIPT] - - self.__add_target_url_option__(args) - self.__add_format_option__(args) - - return self.__generate_command__(args) - - def open_zap_results_file(self): - return read_file(file_path=Constants.REPORT_SCAN_RESULT_FILE) - - def __generate_start_dast_analysis_url__(self) -> str: - url = Constants.URI_START_DAST_ANALYSIS_TEMPLATE_v2.format(soos_base_uri=self.base_uri, - soos_client_id=self.client_id) - - return url - - def __generate_upload_results_url__(self, project_id: str, branch_hash: str, analysis_id: str) -> str: - url = Constants.URI_UPLOAD_DAST_RESULTS_TEMPLATE_v2.format(soos_base_uri=self.base_uri, - soos_client_id=self.client_id, - soos_project_id=project_id, - soos_branch_hash=branch_hash, - soos_analysis_id=analysis_id) - return url - - def __generate_project_details_url__(self, project_id: str) -> str: - url = Constants.URI_PROJECT_DETAILS_TEMPLATE.format(soos_base_uri=self.base_uri, - soos_project_id=project_id) - return url - - def __make_soos_start_analysis_request__(self, command: str) -> DASTStartAnalysisResponse: - message: str = "An error has occurred Starting the Analysis" - try: - log("Making request to SOOS") - api_url: str = self.__generate_start_dast_analysis_url__() - log(f"SOOS URL Endpoint: {api_url}") - - # Validate required fields - if self.project_name is None or len(self.project_name) == 0: - log("projectName is required", LogLevel.ERROR) - sys.exit(1) - - if self.scan_mode is None or len(self.scan_mode) == 0: - log("scanMode is required", LogLevel.ERROR) - sys.exit(1) - - # Obfuscate sensitive data - obfuscated_command = command - if self.auth_bearer_token is not None: - obfuscated_command = obfuscated_command.replace(self.auth_bearer_token, "********") - if self.auth_password is not None: - obfuscated_command = obfuscated_command.replace(self.auth_password, "********") - if self.auth_username is not None: - obfuscated_command = obfuscated_command.replace(self.auth_username, "********") - if self.oauth_token_url is not None: - obfuscated_command = obfuscated_command.replace(self.oauth_token_url, "********") - - param_values: dict = dict( - projectName=self.project_name, - name=datetime.now().strftime("%m/%d/%Y, %H:%M:%S"), - integrationType=self.integration_type, - scriptVersion=SCRIPT_VERSION, - appVersion=self.app_version, - toolName=self.dast_analysis_tool, - toolVersion=self.dast_analysis_tool_version, - commandLine=obfuscated_command, - scanMode=self.scan_mode, - commitHash=self.commit_hash, - branch=self.branch_name, - branchUri=self.branch_uri, - buildVersion=self.build_version, - buildUri=self.build_uri, - operationEnvironment=self.operating_environment or OPERATING_ENVIRONMENT, - integrationName=self.integration_name, - ) - - # Clean up None values - request_body = {k: v for k, v in param_values.items() if v is not None} - - error_response: Optional[Any] = None - - data = json.dumps(request_body) - - api_response: Response = post( - url=api_url, - data=data, - headers={"x-soos-apikey": self.api_key, "Content-Type": Constants.JSON_HEADER_CONTENT_TYPE} - ) - - if api_response.ok: - return DASTStartAnalysisResponse(api_response.json()) - else: - log_error(api_response) - error_response = api_response - log( - "An error has occurred performing the request." - ) - - if error_response is not None: - error_response = error_response.json() - message = error_response["message"] - - except Exception as error: - log(f"Error: {error}") - message = message if message is not None else "An error has occurred Starting the Analysis" - - exit_app(message) - - def __make_soos_scan_status_request__(self, project_id: str, branch_hash: str, - analysis_id: str, status: str, - status_message: Optional[str]) -> bool: - message: str = "An error has occurred Starting the Analysis" - try: - log("Making request to SOOS") - api_url: str = self.__generate_upload_results_url__(project_id, branch_hash, analysis_id) - log(f"SOOS URL Endpoint: {api_url}") - - param_values: dict = dict( - status=status, - message=status_message - ) - - # Clean up None values - request_body = {k: v for k, v in param_values.items() if v is not None} - - error_response: Optional[Any] = None - - data = json.dumps(request_body) - - api_response: Response = patch( - url=api_url, - data=data, - headers={"x-soos-apikey": self.api_key, "Content-Type": Constants.JSON_HEADER_CONTENT_TYPE} - ) - - if api_response.ok: - return True - else: - log_error(api_response) - error_response = api_response - log( - "An error has occurred performing the request" - ) - - if error_response is not None: - error_response = error_response.json() - message = error_response["message"] - - except Exception as error: - log(f"Error: {error}") - message = message if message is not None else "An error has occurred setting the scan status" - self.__make_soos_scan_status_request__(project_id=project_id, - branch_hash=branch_hash, - analysis_id=analysis_id, - status="Error", - status_message=message - ) - - exit_app(message) - - def __make_upload_dast_results_request__( - self, project_id: str, branch_hash: str, analysis_id: str - ) -> bool: - error_response = None - error_message: Optional[str] = None - try: - log("Starting report results processing") - zap_report = self.open_zap_results_file() - log("Making request to SOOS") - api_url: str = self.__generate_upload_results_url__(project_id, branch_hash, analysis_id) - log("SOOS URL Upload Results Endpoint: " + api_url) - results_json = json.loads(zap_report) - log(json.dumps(results_json, indent=2), log_level=LogLevel.DEBUG) - - zap_report_encoded = convert_string_to_b64(json.dumps(results_json)) - files = {"base64Manifest": zap_report_encoded} - - api_response: Response = put( - url=api_url, - data=dict(resultVersion=results_json["@version"]), - files=files, - headers={ - "x-soos-apikey": self.api_key, - "Content_type": Constants.MULTIPART_HEADER_CONTENT_TYPE, - }, - ) - - if api_response.ok: - log("SOOS Upload Success") - return True - else: - error_response = api_response - log_error(error_response) - log("An error has occurred performing the request") - - if error_response is not None: - error_response = error_response.json() - error_message = error_response["message"] - - except Exception as error: - log(f"Error: {error}") - - self.__make_soos_scan_status_request__(project_id=project_id, - branch_hash=branch_hash, - analysis_id=analysis_id, - status="Error", - status_message=error_message - ) - exit_app(error_message) - - def publish_results_to_soos(self, project_id: str, branch_hash: str, analysis_id: str, report_url: str) -> None: - try: - self.__make_upload_dast_results_request__(project_id=project_id, branch_hash=branch_hash, - analysis_id=analysis_id) - - print_line_separator() - log("Report processed successfully") - log(f"Project Id: {project_id}") - log(f"Branch Hash: {branch_hash}") - log(f"Analysis Id: {analysis_id}") - print_line_separator() - log("SOOS DAST Analysis successful") - log(f"Project URL: {report_url}") - print_line_separator() - - except Exception as error: - self.__make_soos_scan_status_request__(project_id=project_id, - branch_hash=branch_hash, - analysis_id=analysis_id, - status="Error", - status_message="An Unexpected error has occurred uploading ZAP Report Results" - ) - exit_app(error) - - def get_analysis_status_soos(self, result_uri): - - analysis_result_response = None - try: - analysis_result_response = requests.get( - url=result_uri, - headers={'x-soos-apikey': self.api_key, 'Content-Type': 'application/json'} - ) - - except Exception as error: - log(f"Analysis Result API Exception Occurred: {error}") - - return analysis_result_response - - def parse_args(self) -> None: - parser = ArgumentParser(description="SOOS DAST") - - # DOCUMENTATION - - parser.add_argument('-hf', "--helpFormatted", dest="help_formatted", - help="Print the --help command in markdown table format", - action="store_false", - default=False, - required=False) - - # SCRIPT PARAMETERS - - parser.add_argument( - "targetURL", - help="Target URL - URL of the site or api to scan. The URL should include the protocol. Ex: https://www.example.com", - ) - parser.add_argument( - "--configFile", - help="Config File - SOOS yaml file with all the configuration for the DAST Analysis (See https://github.com/soos-io/soos-dast#config-file-definition)", - required=False, - ) - parser.add_argument("--clientId", help="SOOS Client ID - get yours from https://app.soos.io/integrate/sca", required=False) - parser.add_argument("--apiKey", help="SOOS API Key - get yours from https://app.soos.io/integrate/sca", required=False) - parser.add_argument("--projectName", help="Project Name - this is what will be displayed in the SOOS app", nargs="+", required=False) - parser.add_argument( - "--scanMode", - help="Scan Mode - Available modes: baseline, fullscan, and apiscan (for more information about scan modes visit https://github.com/soos-io/soos-dast#scan-modes)", - default="baseline", - required=False, - ) - parser.add_argument( - "--apiURL", - help="SOOS API URL - Intended for internal use only, do not modify.", - default="https://api.soos.io/api/", - required=False, - ) - parser.add_argument( - "--debug", - help="Enable to show debug messages.", - default=False, - type=bool, - required=False, - ) - parser.add_argument( - "--ajaxSpider", - help="Ajax Spider - Use the ajax spider in addition to the traditional one. Additional information: https://www.zaproxy.org/docs/desktop/addons/ajax-spider/", - type=bool, - required=False, - ) - parser.add_argument( - "--rules", - help="Rules file to use to INFO, IGNORE or FAIL warnings", - nargs="*", - required=False, - ) - parser.add_argument( - "--contextFile", - help="Context file which will be loaded prior to scanning the target", - nargs="*", - required=False, - ) - parser.add_argument( - "--contextUser", - help="Username to use for authenticated scans - must be defined in the given context file", - nargs="*", - required=False, - ) - parser.add_argument( - "--fullScanMinutes", - help="Number of minutes for the spider to run", - required=False, - ) - parser.add_argument( - "--apiScanFormat", - help="Target API format: OpenAPI, SOAP, or GraphQL", - required=False, - ) - parser.add_argument( - "--level", - help="Log level to show: DEBUG, INFO, WARN, ERROR, CRITICAL", - default="INFO", - required=False, - ) - parser.add_argument( - "--integrationName", - help="Integration Name - Intended for internal use only.", - type=str, - nargs="*", - required=False, - ) - parser.add_argument( - "--integrationType", - help="Integration Type - Intended for internal use only.", - type=str, - nargs="*", - required=False, - ) - parser.add_argument( - "--scriptVersion", - help="Script Version - Intended for internal use only.", - type=str, - nargs="*", - required=False, - ) - parser.add_argument( - "--appVersion", - help="App Version - Intended for internal use only.", - type=str, - nargs="*", - required=False, - ) - parser.add_argument( - "--authDisplay", - help="Minimum level to show: PASS, IGNORE, INFO, WARN or FAIL", - required=False, - ) - parser.add_argument( - "--authUsername", - help="Username to use in auth apps", - required=False, - ) - parser.add_argument( - "--authPassword", - help="Password to use in auth apps", - required=False, - ) - parser.add_argument( - "--authLoginURL", - help="Login url to use in auth apps", - required=False, - ) - parser.add_argument( - "--authUsernameField", - help="Username input id to use in auth apps", - required=False, - ) - parser.add_argument( - "--authPasswordField", - help="Password input id to use in auth apps", - required=False, - ) - parser.add_argument( - "--authSubmitField", - help="Submit button id to use in auth apps", - required=False, - ) - parser.add_argument( - "--authSecondSubmitField", - help="Second submit button id to use in auth apps (for multi-page forms)", - required=False, - ) - parser.add_argument( - "--authSubmitAction", - help="Submit action to perform on form filled. Options: click or submit", - type=str, - required=False, - ) - parser.add_argument( - "--authFormType", - help="simple (all fields are displayed at once), wait_for_password (Password field is displayed only after username is filled), or multi_page (Password field is displayed only after username is filled and submit is clicked)", - type=str, - default="simple", - required=False, - ) - parser.add_argument( - "--authDelayTime", - help="Delay time in seconds to wait for the page to load after performing actions in the form. (Used only on authFormType: wait_for_password and multi_page)", - default=Constants.AUTH_DELAY_TIME, - required=False, - ) - parser.add_argument( - "--zapOptions", - help="Additional ZAP Options", - type=str, - nargs="*", - required=False, - ) - parser.add_argument( - "--requestCookies", - help="Set Cookie values for the requests to the target URL", - type=str, - nargs="*", - default=None, - required=False, - ) - parser.add_argument( - "--requestHeaders", - help="Set extra Header requests", - type=str, - nargs="*", - default=None, - required=False, - ) - parser.add_argument( - "--onFailure", - help="Action to perform when the scan fails. Options: fail_the_build, continue_on_failure", - type=str, - default="continue_on_failure", - required=False, - ) - parser.add_argument( - "--commitHash", - help="The commit hash value from the SCM System", - type=str, - default=None, - required=False, - ) - parser.add_argument( - "--branchName", - help="The name of the branch from the SCM System", - type=str, - default=None, - nargs="*", - required=False, - ) - parser.add_argument( - "--branchURI", - help="The URI to the branch from the SCM System", - default=None, - required=False, - ) - parser.add_argument( - "--buildVersion", - help="Version of application build artifacts", - type=str, - default=None, - required=False, - ) - parser.add_argument( - "--buildURI", - help="URI to CI build info", - type=str, - default=None, - required=False, - ) - parser.add_argument( - "--operatingEnvironment", - help="Set Operating environment for information purposes only", - type=str, - default=None, - nargs="*", - required=False, - ) - parser.add_argument( - "--reportRequestHeaders", - help="Include request/response headers data in report", - type=str, - default="True", - required=False - ) - parser.add_argument( - "--outputFormat", - help="Output format for vulnerabilities: only the value SARIF is available at the moment", - type=str, - default=None, - required=False - ) - parser.add_argument( - "--gpat", - help="GitHub Personal Authorization Token", - type=str, - default=None, - required=False - ) - parser.add_argument( - "--bearerToken", - help="Bearer token to authenticate", - type=str, - default=None, - required=False - ) - parser.add_argument( - "--checkoutDir", - help="Checkout directory to locate SARIF report", - type=str, - default=None, - required=False - ) - parser.add_argument( - "--sarifDestination", - help="SARIF destination to upload report in the form of /", - type=str, - default=None, - required=False - ) - parser.add_argument( - "--sarif", - help="DEPRECATED - SARIF parameter is currently deprecated, please use --outputFormat='sarif' instead", - type=bool, - default=None, - required=False - ) - parser.add_argument( - "--oauthTokenUrl", - help="The authentication URL that grants the access_token.", - type=str, - default=None, - required=False - ) - parser.add_argument( - "--oauthParameters", - help="Parameters to be added to the oauth token request. (eg --oauthParameters=\"client_id:clientID, client_secret:clientSecret, grant_type:client_credentials\")", - type=str, - nargs="*", - default=None, - required=False, - ) - parser.add_argument( - "--updateAddons", - help="Internal use only. Update addons of the zap image.", - type=str, - default="False", - required=False - ) - parser.add_argument( - "--disableRules", - help="Comma separated list of ZAP rules IDs to disable. List for reference https://www.zaproxy.org/docs/alerts/", - nargs="*", - default=None, - required=False - ) - parser.add_argument( - "--otherOptions", - help="Other command line arguments sent directly to the script for items not supported by other command line arguments", - type=str, - nargs="*", - required=False, - ) - - # parse help argument - if "-hf" in sys.argv or "--helpFormatted" in sys.argv: - self.print_help_formatted(parser) - sys.exit(0) - log("Parsing Arguments") - args: Namespace = parser.parse_args() - if args.configFile is not None: - log(f"Reading config file: {args.configFile}", log_level=LogLevel.DEBUG) - file = read_file(file_path=Constants.CONFIG_FILE_FOLDER + args.configFile) - configuration = yaml.load(file, Loader=yaml.FullLoader) - self.parse_configuration(configuration["config"], args.targetURL) - else: - self.parse_configuration(vars(args), args.targetURL) - - def print_help_formatted(self, parser: ArgumentParser): - print("| Argument | Default | Description |") - print("| --- | --- | --- |") - all_rows = [] - for arg, options in parser._option_string_actions.items(): - default_value = options.default - description_text = options.help - all_rows.append(f"| `{'`, `'.join(options.option_strings)}` | {default_value} | {description_text} |") - # remove duplicates - for row in list(OrderedDict.fromkeys(all_rows)): - print(row) - - def run_analysis(self) -> None: - try: - log("Starting SOOS DAST Analysis") - print_line_separator() - - self.parse_args() - - log("Configuration read") - print_line_separator() - - log(f"Project Name: {self.project_name}") - log(f"Scan Mode: {self.scan_mode}") - log(f"API URL: {self.base_uri}") - log(f"Target URL: {self.target_url}") - print_line_separator() - - if self.scan_mode != Constants.API_SCAN: - check: bool = check_site_is_available(self.target_url) - - if check is False: - exit_app(f"The URL {self.target_url} is not available") - return None - - - scan_function = self.scan_mode_map.get(self.scan_mode, None) - - if scan_function is None: - exit_app(f"The scan mode {self.scan_mode} is invalid.") - return None - - log(f"Copying report templates. Include request headers: {self.report_request_headers}", log_level=LogLevel.DEBUG) - os.system("mkdir -p ~/.ZAP/reports") - os.system("mkdir -p /root/.ZAP/reports") - if self.report_request_headers is True: - os.system("cp -R /zap/reports/traditional-json-headers ~/.ZAP/reports/traditional-json") - os.system("cp -R /zap/reports/traditional-json-headers /root/.ZAP/reports/traditional-json") - else: - os.system("cp -R /zap/reports/traditional-json ~/.ZAP/reports/traditional-json") - os.system("cp -R /zap/reports/traditional-json /root/.ZAP/reports/traditional-json") - - command: str = scan_function() - - if self.update_addons: - command = f"{command} --updateAddons" - - if self.zap_options: - command = f"{command} {Constants.ZAP_OPTIONS} \"{self.zap_options}\"" - - if self.other_options: - log(f"Other Options: {str(self.other_options)}") - command = f"{command} {self.other_options}" - - log(f"Executing {self.scan_mode} scan") - soos_dast_start_response = self.__make_soos_start_analysis_request__(command) - - self.__make_soos_scan_status_request__(project_id=soos_dast_start_response.project_id, - branch_hash=soos_dast_start_response.branch_hash, - analysis_id=soos_dast_start_response.analysis_id, - status="Running", - status_message=None - ) - - log(f"Command to be executed: {command}", log_level=LogLevel.DEBUG) - os.system(command) - - run_success = os.path.exists(Constants.REPORT_SCAN_RESULT_FILE) - - print_line_separator() - if run_success is False: - self.__make_soos_scan_status_request__(project_id=soos_dast_start_response.project_id, - branch_hash=soos_dast_start_response.branch_hash, - analysis_id=soos_dast_start_response.analysis_id, - status="Error", - status_message=f"An Unexpected error has occurred running the {self.scan_mode} scan" - ) - raise Exception(f"An Unexpected error has occurred running the {self.scan_mode} scan") - - # Add the discovered urls to the report - discoveredUrls = [] - if (os.path.isfile('./spidered_urls.txt')): - discoveredUrls = open('./spidered_urls.txt', 'r').read().splitlines() - data = json.load(open(Constants.REPORT_SCAN_RESULT_FILE, 'r')) - data['discoveredUrls'] = discoveredUrls - json.dump(data, open(Constants.REPORT_SCAN_RESULT_FILE, 'w')) - - self.publish_results_to_soos( - project_id=soos_dast_start_response.project_id, - branch_hash=soos_dast_start_response.branch_hash, - analysis_id=soos_dast_start_response.analysis_id, - report_url=soos_dast_start_response.scan_url, - ) - - if self.output_format == "sarif": - SOOSSARIFReport.exec(analysis=self, - project_hash=soos_dast_start_response.project_id, - branch_hash=soos_dast_start_response.branch_hash, - scan_id=soos_dast_start_response.analysis_id) - - - while True and self.on_failure == Constants.FAIL_THE_BUILD: - if (datetime.utcnow() - datetime.strptime(ANALYSIS_START_TIME, "%Y-%m-%dT%H:%M:%SZ")).seconds > ANALYSIS_RESULT_MAX_WAIT: - log(f"Analysis Result Max Wait Time Reached ({str(ANALYSIS_RESULT_MAX_WAIT)})") - sys.exit(1) - - analysis_result_api_response = self.get_analysis_status_soos(result_uri=soos_dast_start_response.scan_status_url) - - content_object = analysis_result_api_response.json() - - if analysis_result_api_response.status_code < 299: - analysis_status = str(content_object["status"]) if content_object and "status" in content_object else None - - if analysis_status.lower().startswith("failed") and self.on_failure: - log("Analysis complete - Failures reported") - log("Failing the build.") - sys.exit(1) - elif analysis_status.lower() == "incomplete": - log("Analysis Incomplete. It may have been cancelled or superseded by another scan.") - log("Failing the build.") - sys.exit(1) - elif analysis_status.lower() == "error": - log("Analysis Error.") - log("Failing the build.") - sys.exit(1) - elif analysis_status.lower() == "finished": - return - else: - # Status code that is not pertinent to the result - log(f"Analysis Ongoing. Will retry in {str(ANALYSIS_RESULT_POLLING_INTERVAL)} seconds.") - time.sleep(ANALYSIS_RESULT_POLLING_INTERVAL) - continue - else: - if "message" in analysis_result_api_response.json(): - results_error_code = analysis_result_api_response.json()["code"] - results_error_message = analysis_result_api_response.json()["message"] - log(f"Analysis Results API Status Code: {str(results_error_code)},{results_error_message}") - sys.exit(1) - - - - sys.exit(0) - - except Exception as error: - exit_app(error) - - -class SOOSSARIFReport: - - URL_TEMPLATE = '{soos_base_uri}clients/{clientHash}/projects/{projectHash}/branches/{branchHash}/scan-types/dast/scans/{scanId}/formats/sarif' - GITHUB_URL_TEMPLATE = 'https://api.github.com/repos/{sarif_destination}/code-scanning/sarifs' - - errors_dict = { - 400: "Github: The sarif report is invalid", - 403: "Github: The repository is archived or if github advanced security is not enabled for this repository", - 404: "Github: Resource not found", - 413: "Github: The sarif report is too large", - 503: "Github: Service Unavailable" - } - - def __init__(self): - pass - - @staticmethod - def generate_soos_sarif_url(base_uri: str, client_id: str, project_hash: str, branch_hash: str, - scan_id: str) -> str: - return SOOSSARIFReport.URL_TEMPLATE.format(soos_base_uri=base_uri, - clientHash=client_id, - projectHash=project_hash, - branchHash=branch_hash, - scanId=scan_id) - - @staticmethod - def generate_github_sarif_url(sarif_destination: str) -> str: - return SOOSSARIFReport.GITHUB_URL_TEMPLATE.format(sarif_destination=sarif_destination) - - @staticmethod - def exec(analysis: SOOSDASTAnalysis, project_hash: str, branch_hash: str, - scan_id: str) -> NoReturn: - try: - url = SOOSSARIFReport.generate_soos_sarif_url(base_uri=analysis.base_uri, - client_id=analysis.client_id, - project_hash=project_hash, - branch_hash=branch_hash, - scan_id=scan_id) - - headers = generate_header(api_key=analysis.api_key, content_type="application/json") - sarif_json_response = None - - api_response: requests.Response = requests.get(url=url, headers=headers) - sarif_json_response = handle_response(api_response) - if type(sarif_json_response) is ErrorAPIResponse: - error_message = "A Generate SARIF Report API Exception Occurred." - log(f"{error_message}\n{sarif_json_response.code}-{sarif_json_response.message}") - else: - log("SARIF Report") - log(json.dumps(sarif_json_response, indent=2)) - - if sarif_json_response is None: - log("This project contains no issues. There will be no SARIF upload.") - return - if analysis.github_pat is not None: - sarif_report_str = json.dumps(sarif_json_response) - compressed_sarif_response = base64.b64encode(gzip.compress(bytes(sarif_report_str, 'UTF-8'))) - - github_body_request = { - "commit_sha": analysis.commit_hash, - "ref": analysis.branch_name, - "sarif": compressed_sarif_response.decode(encoding='UTF-8'), - "started_at": ANALYSIS_START_TIME, - "tool_name": "SOOS DAST" - } - - github_sarif_url = SOOSSARIFReport.generate_github_sarif_url(sarif_destination=analysis.sarif_destination) - headers = {"Accept": "application/vnd.github.v3+json", "Authorization": f"token {analysis.github_pat}"} - - log(f"GitHub SARIF URL: {github_sarif_url}") - log(f"GitHub SARIF Header: {str(headers)}") - log(f"GitHub SARIF Body Request") - log(str(json.dumps(github_body_request))) - log("Uploading SARIF Response") - sarif_github_response = requests.post(url=github_sarif_url, data=json.dumps(github_body_request), - headers=headers) - - if sarif_github_response.status_code >= 400: - SOOSSARIFReport.handle_github_sarif_error(status=sarif_github_response.status_code, - json_response=sarif_github_response.json()) - else: - sarif_github_json_response = sarif_github_response.json() - sarif_url = sarif_github_json_response["url"] - sarif_github_status_response = requests.get(url=sarif_url, - headers=headers) - - if sarif_github_status_response.status_code >= 400: - SOOSSARIFReport.handle_github_sarif_error(status=sarif_github_status_response.status_code, - json_response=sarif_github_status_response.json()) - else: - status_json_response = sarif_github_status_response.json() - processing_status = status_json_response["processing_status"] - log("SARIF Report uploaded to GitHub") - log(f"Processing Status: {processing_status}") - if analysis.checkout_dir is not None: - log("Writing sarif file") - sarif_file = open(os.path.join(analysis.checkout_dir, "results.sarif"), "w") - sarif_file.write(json.dumps(sarif_json_response)) - sarif_file.close() - - - except Exception as sarif_exception: - log(f"ERROR: {str(sarif_exception)}") - - @staticmethod - def handle_github_sarif_error(status, json_response): - - error_message = json_response["message"] if json_response is not None and json_response[ - "message"] is not None else SOOSSARIFReport.errors_dict[status] - if error_message is None: - error_message = "An unexpected error has occurred uploading the sarif report to GitHub" - - log(f"ERROR: {error_message}") - - -if __name__ == "__main__": - dastAnalysis = SOOSDASTAnalysis() - dastAnalysis.run_analysis() diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..2cb3c24 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,221 @@ +{ + "name": "soos-dast", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "soos-dast", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@soos-io/api-client": "0.1.5-pre.3", + "argparse": "^2.0.1", + "axios": "^0.27.2", + "tslib": "^2.6.2", + "zaproxy": "^2.0.0-rc.3" + }, + "devDependencies": { + "@types/argparse": "^2.0.11", + "@types/glob": "^8.1.0", + "@types/node": "^20.6.3", + "prettier": "^2.8.8", + "typescript": "^5.2.2" + } + }, + "node_modules/@soos-io/api-client": { + "version": "0.1.5-pre.3", + "resolved": "https://registry.npmjs.org/@soos-io/api-client/-/api-client-0.1.5-pre.3.tgz", + "integrity": "sha512-cTaszKtEquP3pfCb42jKr0HGFS6w0s8afbweAG2hcediWt3SBU3PELP7BMfTgBVITxCYvNukhqBPdSuUowqQKQ==", + "dependencies": { + "axios": "^0.27.2", + "tslib": "^2.6.2" + } + }, + "node_modules/@types/argparse": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/@types/argparse/-/argparse-2.0.12.tgz", + "integrity": "sha512-Qt/6lHaSI+idkJKKTixUTm6q1yjm7EE6ZpsKLkJIHQl7NLAX7lMZRFGAEU8kxaWur3N6L2UE7/U7QR46Isi3vg==", + "dev": true + }, + "node_modules/@types/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==", + "dev": true, + "dependencies": { + "@types/minimatch": "^5.1.2", + "@types/node": "*" + } + }, + "node_modules/@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==", + "dev": true + }, + "node_modules/@types/node": { + "version": "20.8.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.9.tgz", + "integrity": "sha512-UzykFsT3FhHb1h7yD4CA4YhBHq545JC0YnEz41xkipN88eKQtL6rSgocL5tbAP6Ola9Izm/Aw4Ora8He4x0BHg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dependencies": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/prettier": { + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.8.tgz", + "integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, + "node_modules/typescript": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, + "node_modules/zaproxy": { + "version": "2.0.0-rc.3", + "resolved": "https://registry.npmjs.org/zaproxy/-/zaproxy-2.0.0-rc.3.tgz", + "integrity": "sha512-85oymfCSGMy+QG945IgUvwxCLrs24Dci6vMLCcUtldxNEAc87lHhDO7TVfh0cZZ6y5J5ID6bdzt6Yd1QbJk8rA==", + "dependencies": { + "axios": "^1.3.3" + }, + "engines": { + "node": ">=17.0.0" + } + }, + "node_modules/zaproxy/node_modules/axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..4557d2e --- /dev/null +++ b/package.json @@ -0,0 +1,43 @@ +{ + "name": "soos-dast", + "version": "1.0.0", + "description": "SOOS DAST - The affordable no limit web vulnerability scanner", + "main": "index.js", + "scripts": { + "setup:install": "npm install", + "setup:clean-install": "npm ci", + "setup:update": "npx npm-check -u", + "setup:clean": "npx rimraf node_modules && npx rimraf package-lock.json", + "build": "tsc", + "build:clean": "npx rimraf build", + "format": "prettier ./src --check", + "format:fix": "prettier ./src --write", + "typecheck": "tsc --noEmit", + "test:coverage": "npm run test -- --reporter xunit --reporter-option output=ResultsFile.xml", + "check": "npm run format && npm run typecheck && npm run test" + }, + "dependencies": { + "@soos-io/api-client": "0.1.5-pre.4", + "argparse": "^2.0.1", + "axios": "^0.27.2", + "tslib": "^2.6.2", + "zaproxy": "^2.0.0-rc.3" + }, + "devDependencies": { + "@types/argparse": "^2.0.11", + "@types/glob": "^8.1.0", + "@types/node": "^20.6.3", + "prettier": "^2.8.8", + "typescript": "^5.2.2" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/soos-io/soos-dast.git" + }, + "author": "SOOS", + "license": "MIT", + "bugs": { + "url": "https://github.com/soos-io/soos-dast/issues" + }, + "homepage": "https://github.com/soos-io/soos-dast#readme" +} diff --git a/scripts/blindxss.js b/scripts/blindxss.js deleted file mode 100644 index 32aca73..0000000 --- a/scripts/blindxss.js +++ /dev/null @@ -1,74 +0,0 @@ -// An example active scan rule script which uses a set of attack payloads and a set of regexes -// in order to find potential issues. -// Replace or extend the attacks and evidence regexes with you own values. - -// Note that new active scripts will initially be disabled -// Right click the script in the Scripts tree and select "enable" - -// Replace or extend these with your own attacks -// put the attacks you most want to run higher, unless you disable the attack strength check -var attacks = [ - '">', - //'">