From c9b465610a2cc85256baf19948a0d752139be9f1 Mon Sep 17 00:00:00 2001
From: katebygrace <kate.ruh@gmail.com>
Date: Wed, 3 Jan 2024 13:12:41 -0500
Subject: [PATCH] chore: remove sysadmin part2

JIRA:CLOUDSEC-12
---
 devops/jenkins/extend-sandbox-termination.py  |  89 ++++++
 devops/jenkins/mailchimp/courses.csv          |   8 +
 .../jenkins/mailchimp/sync-courselists.bash   |  98 ++++++
 devops/jenkins/saml-ssl-expiration-check.py   |  40 +++
 devops/jenkins/ssl-expiration-check.py        | 106 +++++++
 devops/requirements/base.in                   |  10 +
 devops/requirements/base.txt                  |  56 ++++
 devops/requirements/constraints.txt           |  12 +
 devops/requirements/dev.in                    |  10 +
 devops/requirements/dev.txt                   | 295 ++++++++++++++++++
 devops/requirements/doc.in                    |   9 +
 devops/requirements/doc.txt                   | 212 +++++++++++++
 devops/requirements/pip-tools.in              |   4 +
 devops/requirements/pip-tools.txt             |  20 ++
 devops/requirements/private.readme            |  15 +
 devops/requirements/quality.in                |   9 +
 devops/requirements/quality.txt               | 205 ++++++++++++
 devops/requirements/test.in                   |   8 +
 devops/requirements/test.txt                  | 119 +++++++
 devops/requirements/travis.in                 |   4 +
 devops/requirements/travis.txt                |  20 ++
 devops/resources/saml-ssl-expiration-check.sh |   6 +-
 22 files changed, 1351 insertions(+), 4 deletions(-)
 create mode 100644 devops/jenkins/extend-sandbox-termination.py
 create mode 100644 devops/jenkins/mailchimp/courses.csv
 create mode 100755 devops/jenkins/mailchimp/sync-courselists.bash
 create mode 100644 devops/jenkins/saml-ssl-expiration-check.py
 create mode 100644 devops/jenkins/ssl-expiration-check.py
 create mode 100644 devops/requirements/base.in
 create mode 100644 devops/requirements/base.txt
 create mode 100644 devops/requirements/constraints.txt
 create mode 100644 devops/requirements/dev.in
 create mode 100644 devops/requirements/dev.txt
 create mode 100644 devops/requirements/doc.in
 create mode 100644 devops/requirements/doc.txt
 create mode 100644 devops/requirements/pip-tools.in
 create mode 100644 devops/requirements/pip-tools.txt
 create mode 100644 devops/requirements/private.readme
 create mode 100644 devops/requirements/quality.in
 create mode 100644 devops/requirements/quality.txt
 create mode 100644 devops/requirements/test.in
 create mode 100644 devops/requirements/test.txt
 create mode 100644 devops/requirements/travis.in
 create mode 100644 devops/requirements/travis.txt

diff --git a/devops/jenkins/extend-sandbox-termination.py b/devops/jenkins/extend-sandbox-termination.py
new file mode 100644
index 000000000..6a5740970
--- /dev/null
+++ b/devops/jenkins/extend-sandbox-termination.py
@@ -0,0 +1,89 @@
+__author__ = 'arbab'
+'''
+This script will be used to modify/extend the termination date on the sandbox.
+'''
+import boto
+from datetime import datetime
+from datetime import timedelta
+import logging
+import argparse
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(
+        description="Modify/extend the termination date on the sandbox.")
+
+    parser.add_argument('-n', '--noop', action='store_true',
+                        help="don't actually run the commands", default=False)
+
+    parser.add_argument('-p', '--profile', default=None,
+                        help="AWS profile to use when connecting.")
+
+    extend_group = parser.add_mutually_exclusive_group(required=True)
+
+    extend_group.add_argument('-d', '--day', default=None,
+                        help="number of days", type=int)
+
+    extend_group.add_argument('-a', '--always', default=False,
+                        help="Do not terminate this Sandbox")
+
+    group = parser.add_mutually_exclusive_group(required=True)
+
+    group.add_argument('-u', '--username', default=None,
+                       help="GitHub username")
+
+    group.add_argument('-c', '--custom', default=None,
+                       help="Custom name, if the sandbox was not created with the default options")
+
+    group.add_argument('-i', '--instance-id', default=None,
+                       help="Sandbox Instance ID")
+
+    args = parser.parse_args()
+
+    ec2 = boto.connect_ec2(profile_name=args.profile)
+
+    days_to_increase = args.day
+
+    if args.username:
+        sandbox_name = args.username + '-sandbox'
+        reservations = ec2.get_all_instances(filters={"tag:Name": sandbox_name})
+    if args.custom:
+        sandbox_name = args.custom
+        reservations = ec2.get_all_instances(filters={"tag:Name": sandbox_name})
+    if args.instance_id:
+        instance_id = args.instance_id
+        reservations = ec2.get_all_instances(instance_ids=[instance_id])
+
+    instance = reservations[0].instances[0]
+
+    if args.noop:
+        logger.info("Sandbox ID:{} with Name: {} and Owner: {} will extend by {} days".format(
+                    instance.id,
+                    instance.tags['Name'],
+                    instance.tags['owner'],
+                    days_to_increase
+                    )
+                    )
+    elif args.always:
+        instance.add_tag('do_not_terminate', 'true')
+        logger.info("Sandbox ID:{} with Name: {} and Owner: {} will not be terminate".format(
+                    instance.id,
+                    instance.tags['Name'],
+                    instance.tags['owner'],
+                    )
+                    )
+    else:
+        # modified the terminate time
+        terminate_time = datetime.strptime(str(instance.tags['instance_termination_time']), "%m-%d-%Y %H:%M:%S")
+        terminate_time = terminate_time + timedelta(days=days_to_increase)
+        instance.add_tag('instance_termination_time', terminate_time.strftime("%m-%d-%Y %H:%M:%S"))
+        logger.info("Sandbox ID:{} with Name: {} and Owner: {} has been extended by {} days".format(
+                    instance.id,
+                    instance.tags['Name'],
+                    instance.tags['owner'],
+                    days_to_increase
+                    )
+                    )
\ No newline at end of file
diff --git a/devops/jenkins/mailchimp/courses.csv b/devops/jenkins/mailchimp/courses.csv
new file mode 100644
index 000000000..a6f839477
--- /dev/null
+++ b/devops/jenkins/mailchimp/courses.csv
@@ -0,0 +1,8 @@
+05f99ad611,HarvardX/SW12x/2013_SOND,production,edx
+7dfc19a6ad,HarvardX/SW25x/1T2014,production,edx
+fdb57528e6,HarvardX/SW12.2x/1T2014,production,edx
+bc3991d1aa,HarvardX/SW12.3x/1T2014,production,edx
+295787c2eb,HarvardX/USW30x/2T2014,production,edx
+d20f7569e3,HarvardX/SW12.4x/1T2014,production,edx
+0f52d660fa,HarvardX/SW12.5x/2T2014,production,edx
+22a674a898,HarvardX/SW12.6x/2T2014,production,edx
diff --git a/devops/jenkins/mailchimp/sync-courselists.bash b/devops/jenkins/mailchimp/sync-courselists.bash
new file mode 100755
index 000000000..9560e7d85
--- /dev/null
+++ b/devops/jenkins/mailchimp/sync-courselists.bash
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -x
+
+if [[ -z $WORKSPACE ]]; then
+    echo "Environment incorrect for this wrapper script"
+    env
+    exit 1
+fi
+
+cd "$WORKSPACE/edx-platform"
+
+# install requirements
+# These requirements will be installed into the shinginpanda
+# virtualenv on the jenkins server and are necessary to run
+# management commands locally.
+
+pip install --exists-action w -r requirements/edx/pre.txt
+pip install --exists-action w -r requirements/edx/base.txt
+pip install --exists-action w -r requirements/edx/post.txt
+pip install --exists-action w -r requirements/edx/github.txt
+pip install --exists-action w -r requirements/edx/local.txt
+
+cd "$WORKSPACE/configuration"
+
+pip install --exist-action w -r pre-requirements.txt
+pip install --exist-action w -r requirements.txt
+
+cd "$WORKSPACE/configuration/playbooks"
+
+if [[ -f ${WORKSPACE}/configuration-secure/ansible/vars/${deployment}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/${deployment}.yml"
+fi
+
+if [[ -f ${WORKSPACE}/configuration-secure/ansible/vars/${environment}-${deployment}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/${environment}-${deployment}.yml"
+fi
+
+for extra_var in $extra_vars; do
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/$extra_var"
+done
+
+extra_var_args+=" -e edxapp_app_dir=${WORKSPACE}"
+extra_var_args+=" -e EDXAPP_CFG_DIR=${WORKSPACE}"
+extra_var_args+=" -e edxapp_code_dir=${WORKSPACE}/edx-platform"
+extra_var_args+=" -e edxapp_user=jenkins"
+
+# Generate the json configuration files
+ansible-playbook -c local $extra_var_args --tags edxapp_cfg -i localhost, -s -U jenkins edxapp.yml
+
+# Run migrations and replace literal '\n' with actual newlines to make the output
+# easier to read
+
+
+EDX_PATH="${WORKSPACE}/edx-platform"
+#DJANGO_ADMIN="${JENKINS_HOME}/.virtualenvs/mailchimp/bin/django-admin.py"
+DJANGO_ADMIN="${VIRTUAL_ENV}/bin/python ${EDX_PATH}/manage.py lms --settings=production"
+
+get_key () {
+  case $1 in
+      "edx"       ) ORG_KEY="7b87ccd203b973d87d0ac4423192afa6-us5";;
+  esac
+  echo $ORG_KEY
+}
+
+
+sync_announcements () {
+    LIST_ID=$1
+    CONFIGURATION=$2
+    ORG_NAME=$3
+    ORG_KEY=$(get_key ${ORG_NAME})
+    CMD="${DJANGO_ADMIN} mailchimp_sync_announcements --key=${ORG_KEY} --list=${LIST_ID}"
+    $CMD
+}
+
+sync_course () {
+    LIST_ID=$1
+    COURSE_ID=$2
+    CONFIGURATION=$3
+    ORG_NAME=$4
+    ORG_KEY=$(get_key ${ORG_NAME})
+    SEGMENTS=${5:-0}
+    CMD="${DJANGO_ADMIN} mailchimp_sync_course --key=${ORG_KEY} --list=${LIST_ID} --course=${COURSE_ID} --segments=${SEGMENTS}"
+    $CMD
+}
+
+OLD_IFS=${IFS}
+
+cd ${WORKSPACE}/sysadmin/jenkins/mailchimp
+
+while read -r line
+do
+    IFS=","
+    set $line
+    IFS=${OLD_IFS}
+     sync_course $1 $2 $3 $4
+done < "courses.csv"
+
+sync_announcements 237694b56d production edx
diff --git a/devops/jenkins/saml-ssl-expiration-check.py b/devops/jenkins/saml-ssl-expiration-check.py
new file mode 100644
index 000000000..124e7ec3c
--- /dev/null
+++ b/devops/jenkins/saml-ssl-expiration-check.py
@@ -0,0 +1,40 @@
+import argparse
+import logging
+import OpenSSL
+from datetime import datetime, timedelta
+import sys
+import yaml
+from os.path import basename
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-e', '--region', default='us-east-1', required=True,
+                        help="AWS Region")
+
+    parser.add_argument('-d', '--days', type=int,
+                        help="Alert if SSL certificate will expire within these days", default=90)
+    parser.add_argument('-i','--file',
+                        help="input YAML file to parse and get SAML cert")
+    
+
+    args = parser.parse_args()
+
+    time_now = datetime.now()
+    ssl_expire_check = time_now + timedelta(days=args.days)
+    saml_cert_file = args.file
+    expired_ssl =  basename(saml_cert_file).strip('.yml')
+
+    with open(saml_cert_file) as f:
+        secure_config = yaml.safe_load(f)
+    cert = secure_config['EDXAPP_SOCIAL_AUTH_SAML_SP_PUBLIC_CERT']
+    x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
+    cert_expire_date = datetime.strptime(x509.get_notAfter().decode('utf-8'), "%Y%m%d%H%M%S%fZ").date()
+
+    if ssl_expire_check.date() > cert_expire_date:
+        logger.info("{} SAML certificate will be expired on  {}".format(expired_ssl,cert_expire_date))
+        sys.exit(1)
diff --git a/devops/jenkins/ssl-expiration-check.py b/devops/jenkins/ssl-expiration-check.py
new file mode 100644
index 000000000..3effa5c1f
--- /dev/null
+++ b/devops/jenkins/ssl-expiration-check.py
@@ -0,0 +1,106 @@
+import boto3
+import argparse
+import logging
+import ssl
+import OpenSSL
+import smtplib
+from datetime import date, datetime, timedelta
+from socket import socket
+from pprint import pformat
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def send_an_email(toaddr, fromaddr, expired_ssl_message, not_check_message, region):
+    client = boto3.client('ses', region_name=region)
+
+    message = """
+
+    <p>Hello,</p>
+
+    <p>Certificates that are associated with these load-balancers will be expired in next 30 days: </p>
+
+    <p> {expired_ssl_message} </p>
+
+    <p> These ELBs have SSL certificate but don't have any instance associated to them: </p>
+
+    <p> {not_check_message} </p>
+
+    """.format(expired_ssl_message=expired_ssl_message, not_check_message=not_check_message)
+    client.send_email(
+        Source=fromaddr,
+        Destination={
+            'ToAddresses': [
+                toaddr
+            ]
+        },
+        Message={
+            'Subject': {
+                'Data': 'These Certificates will be expired in the next 30 days',
+                'Charset': 'utf-8'
+            },
+            'Body': {
+                'Html':{
+                    'Data': message,
+                    'Charset': 'utf-8'
+                }
+            }
+        }
+    )
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(
+        description="Find the SSL Certificates that will expire after X days.")
+
+    parser.add_argument('-e', '--region', default='us-east-1', required=True,
+                        help="AWS Region for getting the records", type=str)
+
+    parser.add_argument('-d', '--days', type=int,
+                        help="Alert if SSL certificate will expire within these days", default=30)
+    
+    email_args = parser.add_argument_group("Email Arguments",
+                                                "Args for sending email.")
+
+    email_args.add_argument('-r', '--recipient', type=str,
+                        help='Recipient email address')
+
+    email_args.add_argument('-f', '--from-email', type=str,
+                                 help="Sender email address for email notifications. "
+                                      "Email notifications will be disabled if not provided")
+
+    args = parser.parse_args()
+
+    expire_ssl = []
+    time_now = datetime.now()
+    ssl_expire_check = time_now + timedelta(days=args.days)
+    
+    elb_conn = boto3.client('elb', region_name=args.region)
+    elbs = elb_conn.describe_load_balancers()['LoadBalancerDescriptions']
+
+    elbs_with_ssl = [elb for elb in elbs for listener in elb['ListenerDescriptions'] if (listener['Listener']['LoadBalancerPort'] == 443)]
+
+    elbs_to_check = [(elb['LoadBalancerName'],elb['DNSName']) for elb in elbs_with_ssl if elb['Instances']]
+
+    elbs_not_need_to_check = [elb['DNSName'] for elb in elbs_with_ssl if not elb['Instances']]
+
+    for elb in elbs_to_check:
+        elb_tags = elb_conn.describe_tags(LoadBalancerNames=[elb[0]])['TagDescriptions'][0]['Tags']
+        for tag in elb_tags:
+          if 'kubernetes.io' in tag["Key"]:
+              break
+        else:
+            print("Checking {}".format(elb[1]))
+            cert = ssl.get_server_certificate((elb[1], 443))
+            x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
+            cert_expire_date = datetime.strptime(x509.get_notAfter().decode(), "%Y%m%d%H%M%S%fZ").date()
+            if ssl_expire_check.date() > cert_expire_date:
+                print("Expires {}".format(cert_expire_date))
+                expire_ssl.append((elb[1],cert_expire_date))
+
+    if expire_ssl or elbs_not_need_to_check:
+        expired_ssl_message = pformat(expire_ssl)
+        not_check_message = pformat(elbs_not_need_to_check)
+        print(not_check_message)
+        if args.from_email and args.recipient:
+            send_an_email(args.recipient, args.from_email, expired_ssl_message, not_check_message, args.region)
diff --git a/devops/requirements/base.in b/devops/requirements/base.in
new file mode 100644
index 000000000..5f7f34712
--- /dev/null
+++ b/devops/requirements/base.in
@@ -0,0 +1,10 @@
+# Core requirements for using this application
+-c constraints.txt
+boto
+boto3
+python-gnupg
+pyyaml
+pymysql
+pymongo
+requests
+pyOpenSSL
diff --git a/devops/requirements/base.txt b/devops/requirements/base.txt
new file mode 100644
index 000000000..a5f3f3491
--- /dev/null
+++ b/devops/requirements/base.txt
@@ -0,0 +1,56 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+boto==2.49.0
+    # via -r requirements/base.in
+boto3==1.20.26
+    # via -r requirements/base.in
+botocore==1.23.26
+    # via
+    #   boto3
+    #   s3transfer
+certifi==2021.10.8
+    # via requests
+cffi==1.15.0
+    # via cryptography
+charset-normalizer==2.0.9
+    # via requests
+cryptography==3.4.8
+    # via
+    #   -c requirements/constraints.txt
+    #   pyopenssl
+idna==3.3
+    # via requests
+jmespath==0.10.0
+    # via
+    #   boto3
+    #   botocore
+pycparser==2.21
+    # via cffi
+pymongo==4.0.1
+    # via -r requirements/base.in
+pymysql==1.0.2
+    # via -r requirements/base.in
+pyopenssl==21.0.0
+    # via -r requirements/base.in
+python-dateutil==2.8.2
+    # via botocore
+python-gnupg==0.4.8
+    # via -r requirements/base.in
+pyyaml==6.0
+    # via -r requirements/base.in
+requests==2.26.0
+    # via -r requirements/base.in
+s3transfer==0.5.0
+    # via boto3
+six==1.16.0
+    # via
+    #   pyopenssl
+    #   python-dateutil
+urllib3==1.26.7
+    # via
+    #   botocore
+    #   requests
diff --git a/devops/requirements/constraints.txt b/devops/requirements/constraints.txt
new file mode 100644
index 000000000..4386c3e6d
--- /dev/null
+++ b/devops/requirements/constraints.txt
@@ -0,0 +1,12 @@
+# Version constraints for pip-installation.
+#
+# This file doesn't install any packages. It specifies version constraints
+# that will be applied if a package is needed.
+#
+# When pinning something here, please provide an explanation of why. Ideally,
+# link to other information that will help people in the future to remove the
+# pin when possible.  Writing an issue against the offending project and
+# linking to it here is good.
+
+# Pin cryptography because it broke saml-ssl-expiration-check.py, unpin when fixed
+cryptography<=3.4.8
diff --git a/devops/requirements/dev.in b/devops/requirements/dev.in
new file mode 100644
index 000000000..2e8c93c69
--- /dev/null
+++ b/devops/requirements/dev.in
@@ -0,0 +1,10 @@
+# Additional requirements for development of this application
+-c constraints.txt
+
+-r pip-tools.txt          # pip-tools and its dependencies, for managing requirements files
+-r quality.txt            # Core and quality check dependencies
+-r travis.txt             # tox and related dependencies
+
+diff-cover                # Changeset diff test coverage
+edx-i18n-tools            # For i18n_tool dummy
+tox-battery               # Makes tox aware of requirements file changes
diff --git a/devops/requirements/dev.txt b/devops/requirements/dev.txt
new file mode 100644
index 000000000..dc5768fba
--- /dev/null
+++ b/devops/requirements/dev.txt
@@ -0,0 +1,295 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+asgiref==3.4.1
+    # via django
+astroid==2.9.0
+    # via
+    #   -r requirements/quality.txt
+    #   pylint
+    #   pylint-celery
+attrs==21.2.0
+    # via
+    #   -r requirements/quality.txt
+    #   pytest
+backports.entry-points-selectable==1.1.1
+    # via virtualenv
+boto==2.49.0
+    # via -r requirements/quality.txt
+boto3==1.20.26
+    # via -r requirements/quality.txt
+botocore==1.23.26
+    # via
+    #   -r requirements/quality.txt
+    #   boto3
+    #   s3transfer
+certifi==2021.10.8
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/travis.txt
+    #   requests
+cffi==1.15.0
+    # via
+    #   -r requirements/quality.txt
+    #   cryptography
+chardet==4.0.0
+    # via diff-cover
+charset-normalizer==2.0.9
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/travis.txt
+    #   requests
+click==8.0.3
+    # via
+    #   -r requirements/pip-tools.txt
+    #   -r requirements/quality.txt
+    #   click-log
+    #   code-annotations
+    #   edx-lint
+    #   pip-tools
+click-log==0.3.2
+    # via
+    #   -r requirements/quality.txt
+    #   edx-lint
+code-annotations==1.2.0
+    # via
+    #   -r requirements/quality.txt
+    #   edx-lint
+codecov==2.1.12
+    # via -r requirements/travis.txt
+coverage[toml]==6.2
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/travis.txt
+    #   codecov
+    #   pytest-cov
+cryptography==3.4.8
+    # via
+    #   -c requirements/constraints.txt
+    #   -r requirements/quality.txt
+    #   pyopenssl
+diff-cover==6.4.4
+    # via -r requirements/dev.in
+distlib==0.3.4
+    # via virtualenv
+django==3.2.10
+    # via edx-i18n-tools
+edx-i18n-tools==0.8.1
+    # via -r requirements/dev.in
+edx-lint==5.2.1
+    # via -r requirements/quality.txt
+filelock==3.4.0
+    # via
+    #   tox
+    #   virtualenv
+idna==3.3
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/travis.txt
+    #   requests
+iniconfig==1.1.1
+    # via
+    #   -r requirements/quality.txt
+    #   pytest
+isort==5.10.1
+    # via
+    #   -r requirements/quality.txt
+    #   pylint
+jinja2==3.0.3
+    # via
+    #   -r requirements/quality.txt
+    #   code-annotations
+    #   diff-cover
+jmespath==0.10.0
+    # via
+    #   -r requirements/quality.txt
+    #   boto3
+    #   botocore
+lazy-object-proxy==1.7.1
+    # via
+    #   -r requirements/quality.txt
+    #   astroid
+markupsafe==2.0.1
+    # via
+    #   -r requirements/quality.txt
+    #   jinja2
+mccabe==0.6.1
+    # via
+    #   -r requirements/quality.txt
+    #   pylint
+packaging==21.3
+    # via
+    #   -r requirements/quality.txt
+    #   pytest
+    #   tox
+path==16.2.0
+    # via edx-i18n-tools
+pbr==5.8.0
+    # via
+    #   -r requirements/quality.txt
+    #   stevedore
+pep517==0.12.0
+    # via
+    #   -r requirements/pip-tools.txt
+    #   pip-tools
+pip-tools==6.4.0
+    # via -r requirements/pip-tools.txt
+platformdirs==2.4.0
+    # via
+    #   -r requirements/quality.txt
+    #   pylint
+    #   virtualenv
+pluggy==1.0.0
+    # via
+    #   -r requirements/quality.txt
+    #   diff-cover
+    #   pytest
+    #   tox
+polib==1.1.1
+    # via edx-i18n-tools
+py==1.11.0
+    # via
+    #   -r requirements/quality.txt
+    #   pytest
+    #   tox
+pycodestyle==2.8.0
+    # via -r requirements/quality.txt
+pycparser==2.21
+    # via
+    #   -r requirements/quality.txt
+    #   cffi
+pydocstyle==6.1.1
+    # via -r requirements/quality.txt
+pygments==2.10.0
+    # via diff-cover
+pylint==2.12.2
+    # via
+    #   -r requirements/quality.txt
+    #   edx-lint
+    #   pylint-celery
+    #   pylint-django
+    #   pylint-plugin-utils
+pylint-celery==0.3
+    # via
+    #   -r requirements/quality.txt
+    #   edx-lint
+pylint-django==2.4.4
+    # via
+    #   -r requirements/quality.txt
+    #   edx-lint
+pylint-plugin-utils==0.6
+    # via
+    #   -r requirements/quality.txt
+    #   pylint-celery
+    #   pylint-django
+pymongo==4.0.1
+    # via -r requirements/quality.txt
+pymysql==1.0.2
+    # via -r requirements/quality.txt
+pyopenssl==21.0.0
+    # via -r requirements/quality.txt
+pyparsing==3.0.6
+    # via
+    #   -r requirements/quality.txt
+    #   packaging
+pytest==6.2.5
+    # via
+    #   -r requirements/quality.txt
+    #   pytest-cov
+    #   pytest-django
+pytest-cov==3.0.0
+    # via -r requirements/quality.txt
+pytest-django==4.5.2
+    # via -r requirements/quality.txt
+python-dateutil==2.8.2
+    # via
+    #   -r requirements/quality.txt
+    #   botocore
+python-gnupg==0.4.8
+    # via -r requirements/quality.txt
+python-slugify==5.0.2
+    # via
+    #   -r requirements/quality.txt
+    #   code-annotations
+pytz==2021.3
+    # via django
+pyyaml==6.0
+    # via
+    #   -r requirements/quality.txt
+    #   code-annotations
+    #   edx-i18n-tools
+requests==2.26.0
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/travis.txt
+    #   codecov
+s3transfer==0.5.0
+    # via
+    #   -r requirements/quality.txt
+    #   boto3
+six==1.16.0
+    # via
+    #   -r requirements/quality.txt
+    #   edx-lint
+    #   pyopenssl
+    #   python-dateutil
+    #   tox
+    #   virtualenv
+snowballstemmer==2.2.0
+    # via
+    #   -r requirements/quality.txt
+    #   pydocstyle
+sqlparse==0.4.2
+    # via django
+stevedore==3.5.0
+    # via
+    #   -r requirements/quality.txt
+    #   code-annotations
+text-unidecode==1.3
+    # via
+    #   -r requirements/quality.txt
+    #   python-slugify
+toml==0.10.2
+    # via
+    #   -r requirements/quality.txt
+    #   pylint
+    #   pytest
+    #   tox
+tomli==2.0.0
+    # via
+    #   -r requirements/pip-tools.txt
+    #   -r requirements/quality.txt
+    #   coverage
+    #   pep517
+tox==3.24.4
+    # via tox-battery
+tox-battery==0.6.1
+    # via -r requirements/dev.in
+typing-extensions==4.0.1
+    # via
+    #   -r requirements/quality.txt
+    #   astroid
+    #   pylint
+urllib3==1.26.7
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/travis.txt
+    #   botocore
+    #   requests
+virtualenv==20.10.0
+    # via tox
+wheel==0.37.1
+    # via
+    #   -r requirements/pip-tools.txt
+    #   pip-tools
+wrapt==1.13.3
+    # via
+    #   -r requirements/quality.txt
+    #   astroid
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools
diff --git a/devops/requirements/doc.in b/devops/requirements/doc.in
new file mode 100644
index 000000000..690e8e1d5
--- /dev/null
+++ b/devops/requirements/doc.in
@@ -0,0 +1,9 @@
+# Requirements for documentation validation
+-c constraints.txt
+
+-r test.txt               # Core and testing dependencies for this package
+
+doc8                      # reStructuredText style checker
+edx_sphinx_theme          # edX theme for Sphinx output
+readme_renderer           # Validates README.rst for usage on PyPI
+Sphinx                    # Documentation builder
diff --git a/devops/requirements/doc.txt b/devops/requirements/doc.txt
new file mode 100644
index 000000000..2942cd633
--- /dev/null
+++ b/devops/requirements/doc.txt
@@ -0,0 +1,212 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+alabaster==0.7.12
+    # via sphinx
+attrs==21.2.0
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+babel==2.9.1
+    # via sphinx
+bleach==4.1.0
+    # via readme-renderer
+boto==2.49.0
+    # via -r requirements/test.txt
+boto3==1.20.26
+    # via -r requirements/test.txt
+botocore==1.23.26
+    # via
+    #   -r requirements/test.txt
+    #   boto3
+    #   s3transfer
+certifi==2021.10.8
+    # via
+    #   -r requirements/test.txt
+    #   requests
+cffi==1.15.0
+    # via
+    #   -r requirements/test.txt
+    #   cryptography
+charset-normalizer==2.0.9
+    # via
+    #   -r requirements/test.txt
+    #   requests
+click==8.0.3
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+code-annotations==1.2.0
+    # via -r requirements/test.txt
+coverage[toml]==6.2
+    # via
+    #   -r requirements/test.txt
+    #   pytest-cov
+cryptography==3.4.8
+    # via
+    #   -c requirements/constraints.txt
+    #   -r requirements/test.txt
+    #   pyopenssl
+doc8==0.10.1
+    # via -r requirements/doc.in
+docutils==0.17.1
+    # via
+    #   doc8
+    #   readme-renderer
+    #   restructuredtext-lint
+    #   sphinx
+edx-sphinx-theme==3.0.0
+    # via -r requirements/doc.in
+idna==3.3
+    # via
+    #   -r requirements/test.txt
+    #   requests
+imagesize==1.3.0
+    # via sphinx
+iniconfig==1.1.1
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+jinja2==3.0.3
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+    #   sphinx
+jmespath==0.10.0
+    # via
+    #   -r requirements/test.txt
+    #   boto3
+    #   botocore
+markupsafe==2.0.1
+    # via
+    #   -r requirements/test.txt
+    #   jinja2
+packaging==21.3
+    # via
+    #   -r requirements/test.txt
+    #   bleach
+    #   pytest
+    #   sphinx
+pbr==5.8.0
+    # via
+    #   -r requirements/test.txt
+    #   stevedore
+pluggy==1.0.0
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+py==1.11.0
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+pycparser==2.21
+    # via
+    #   -r requirements/test.txt
+    #   cffi
+pygments==2.10.0
+    # via
+    #   doc8
+    #   readme-renderer
+    #   sphinx
+pymongo==4.0.1
+    # via -r requirements/test.txt
+pymysql==1.0.2
+    # via -r requirements/test.txt
+pyopenssl==21.0.0
+    # via -r requirements/test.txt
+pyparsing==3.0.6
+    # via
+    #   -r requirements/test.txt
+    #   packaging
+pytest==6.2.5
+    # via
+    #   -r requirements/test.txt
+    #   pytest-cov
+    #   pytest-django
+pytest-cov==3.0.0
+    # via -r requirements/test.txt
+pytest-django==4.5.2
+    # via -r requirements/test.txt
+python-dateutil==2.8.2
+    # via
+    #   -r requirements/test.txt
+    #   botocore
+python-gnupg==0.4.8
+    # via -r requirements/test.txt
+python-slugify==5.0.2
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+pytz==2021.3
+    # via babel
+pyyaml==6.0
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+readme-renderer==32.0
+    # via -r requirements/doc.in
+requests==2.26.0
+    # via
+    #   -r requirements/test.txt
+    #   sphinx
+restructuredtext-lint==1.3.2
+    # via doc8
+s3transfer==0.5.0
+    # via
+    #   -r requirements/test.txt
+    #   boto3
+six==1.16.0
+    # via
+    #   -r requirements/test.txt
+    #   bleach
+    #   edx-sphinx-theme
+    #   pyopenssl
+    #   python-dateutil
+snowballstemmer==2.2.0
+    # via sphinx
+sphinx==4.3.2
+    # via
+    #   -r requirements/doc.in
+    #   edx-sphinx-theme
+sphinxcontrib-applehelp==1.0.2
+    # via sphinx
+sphinxcontrib-devhelp==1.0.2
+    # via sphinx
+sphinxcontrib-htmlhelp==2.0.0
+    # via sphinx
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==1.0.3
+    # via sphinx
+sphinxcontrib-serializinghtml==1.1.5
+    # via sphinx
+stevedore==3.5.0
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+    #   doc8
+text-unidecode==1.3
+    # via
+    #   -r requirements/test.txt
+    #   python-slugify
+toml==0.10.2
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+tomli==2.0.0
+    # via
+    #   -r requirements/test.txt
+    #   coverage
+urllib3==1.26.7
+    # via
+    #   -r requirements/test.txt
+    #   botocore
+    #   requests
+webencodings==0.5.1
+    # via bleach
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/devops/requirements/pip-tools.in b/devops/requirements/pip-tools.in
new file mode 100644
index 000000000..3f1b64ae9
--- /dev/null
+++ b/devops/requirements/pip-tools.in
@@ -0,0 +1,4 @@
+# Just the dependencies to run pip-tools, mainly for the "upgrade" make target
+-c constraints.txt
+
+pip-tools                 # Contains pip-compile, used to generate pip requirements files
diff --git a/devops/requirements/pip-tools.txt b/devops/requirements/pip-tools.txt
new file mode 100644
index 000000000..f4b492113
--- /dev/null
+++ b/devops/requirements/pip-tools.txt
@@ -0,0 +1,20 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+click==8.0.3
+    # via pip-tools
+pep517==0.12.0
+    # via pip-tools
+pip-tools==6.4.0
+    # via -r requirements/pip-tools.in
+tomli==2.0.0
+    # via pep517
+wheel==0.37.1
+    # via pip-tools
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools
diff --git a/devops/requirements/private.readme b/devops/requirements/private.readme
new file mode 100644
index 000000000..5600a1075
--- /dev/null
+++ b/devops/requirements/private.readme
@@ -0,0 +1,15 @@
+# If there are any Python packages you want to keep in your virtualenv beyond
+# those listed in the official requirements files, create a "private.in" file
+# and list them there.  Generate the corresponding "private.txt" file pinning
+# all of their indirect dependencies to specific versions as follows:
+
+#     pip-compile private.in
+
+# This allows you to use "pip-sync" without removing these packages:
+
+#     pip-sync requirements/*.txt
+
+# "private.in" and "private.txt" aren't checked into git to avoid merge
+# conflicts, and the presence of this file allows "private.*" to be
+# included in scripted pip-sync usage without requiring that those files be
+# created first.
diff --git a/devops/requirements/quality.in b/devops/requirements/quality.in
new file mode 100644
index 000000000..0bd84a646
--- /dev/null
+++ b/devops/requirements/quality.in
@@ -0,0 +1,9 @@
+# Requirements for code quality checks
+-c constraints.txt
+
+-r test.txt               # Core and testing dependencies for this package
+
+edx-lint                  # edX pylint rules and plugins
+isort                     # to standardize order of imports
+pycodestyle               # PEP 8 compliance validation
+pydocstyle                # PEP 257 compliance validation
diff --git a/devops/requirements/quality.txt b/devops/requirements/quality.txt
new file mode 100644
index 000000000..86f5ed5fa
--- /dev/null
+++ b/devops/requirements/quality.txt
@@ -0,0 +1,205 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+astroid==2.9.0
+    # via
+    #   pylint
+    #   pylint-celery
+attrs==21.2.0
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+boto==2.49.0
+    # via -r requirements/test.txt
+boto3==1.20.26
+    # via -r requirements/test.txt
+botocore==1.23.26
+    # via
+    #   -r requirements/test.txt
+    #   boto3
+    #   s3transfer
+certifi==2021.10.8
+    # via
+    #   -r requirements/test.txt
+    #   requests
+cffi==1.15.0
+    # via
+    #   -r requirements/test.txt
+    #   cryptography
+charset-normalizer==2.0.9
+    # via
+    #   -r requirements/test.txt
+    #   requests
+click==8.0.3
+    # via
+    #   -r requirements/test.txt
+    #   click-log
+    #   code-annotations
+    #   edx-lint
+click-log==0.3.2
+    # via edx-lint
+code-annotations==1.2.0
+    # via
+    #   -r requirements/test.txt
+    #   edx-lint
+coverage[toml]==6.2
+    # via
+    #   -r requirements/test.txt
+    #   pytest-cov
+cryptography==3.4.8
+    # via
+    #   -c requirements/constraints.txt
+    #   -r requirements/test.txt
+    #   pyopenssl
+edx-lint==5.2.1
+    # via -r requirements/quality.in
+idna==3.3
+    # via
+    #   -r requirements/test.txt
+    #   requests
+iniconfig==1.1.1
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+isort==5.10.1
+    # via
+    #   -r requirements/quality.in
+    #   pylint
+jinja2==3.0.3
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+jmespath==0.10.0
+    # via
+    #   -r requirements/test.txt
+    #   boto3
+    #   botocore
+lazy-object-proxy==1.7.1
+    # via astroid
+markupsafe==2.0.1
+    # via
+    #   -r requirements/test.txt
+    #   jinja2
+mccabe==0.6.1
+    # via pylint
+packaging==21.3
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+pbr==5.8.0
+    # via
+    #   -r requirements/test.txt
+    #   stevedore
+platformdirs==2.4.0
+    # via pylint
+pluggy==1.0.0
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+py==1.11.0
+    # via
+    #   -r requirements/test.txt
+    #   pytest
+pycodestyle==2.8.0
+    # via -r requirements/quality.in
+pycparser==2.21
+    # via
+    #   -r requirements/test.txt
+    #   cffi
+pydocstyle==6.1.1
+    # via -r requirements/quality.in
+pylint==2.12.2
+    # via
+    #   edx-lint
+    #   pylint-celery
+    #   pylint-django
+    #   pylint-plugin-utils
+pylint-celery==0.3
+    # via edx-lint
+pylint-django==2.4.4
+    # via edx-lint
+pylint-plugin-utils==0.6
+    # via
+    #   pylint-celery
+    #   pylint-django
+pymongo==4.0.1
+    # via -r requirements/test.txt
+pymysql==1.0.2
+    # via -r requirements/test.txt
+pyopenssl==21.0.0
+    # via -r requirements/test.txt
+pyparsing==3.0.6
+    # via
+    #   -r requirements/test.txt
+    #   packaging
+pytest==6.2.5
+    # via
+    #   -r requirements/test.txt
+    #   pytest-cov
+    #   pytest-django
+pytest-cov==3.0.0
+    # via -r requirements/test.txt
+pytest-django==4.5.2
+    # via -r requirements/test.txt
+python-dateutil==2.8.2
+    # via
+    #   -r requirements/test.txt
+    #   botocore
+python-gnupg==0.4.8
+    # via -r requirements/test.txt
+python-slugify==5.0.2
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+pyyaml==6.0
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+requests==2.26.0
+    # via -r requirements/test.txt
+s3transfer==0.5.0
+    # via
+    #   -r requirements/test.txt
+    #   boto3
+six==1.16.0
+    # via
+    #   -r requirements/test.txt
+    #   edx-lint
+    #   pyopenssl
+    #   python-dateutil
+snowballstemmer==2.2.0
+    # via pydocstyle
+stevedore==3.5.0
+    # via
+    #   -r requirements/test.txt
+    #   code-annotations
+text-unidecode==1.3
+    # via
+    #   -r requirements/test.txt
+    #   python-slugify
+toml==0.10.2
+    # via
+    #   -r requirements/test.txt
+    #   pylint
+    #   pytest
+tomli==2.0.0
+    # via
+    #   -r requirements/test.txt
+    #   coverage
+typing-extensions==4.0.1
+    # via
+    #   astroid
+    #   pylint
+urllib3==1.26.7
+    # via
+    #   -r requirements/test.txt
+    #   botocore
+    #   requests
+wrapt==1.13.3
+    # via astroid
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/devops/requirements/test.in b/devops/requirements/test.in
new file mode 100644
index 000000000..6797160bf
--- /dev/null
+++ b/devops/requirements/test.in
@@ -0,0 +1,8 @@
+# Requirements for test runs.
+-c constraints.txt
+
+-r base.txt               # Core dependencies for this package
+
+pytest-cov                # pytest extension for code coverage statistics
+pytest-django             # pytest extension for better Django support
+code-annotations          # provides commands used by the pii_check make target.
diff --git a/devops/requirements/test.txt b/devops/requirements/test.txt
new file mode 100644
index 000000000..5838cfa46
--- /dev/null
+++ b/devops/requirements/test.txt
@@ -0,0 +1,119 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+attrs==21.2.0
+    # via pytest
+boto==2.49.0
+    # via -r requirements/base.txt
+boto3==1.20.26
+    # via -r requirements/base.txt
+botocore==1.23.26
+    # via
+    #   -r requirements/base.txt
+    #   boto3
+    #   s3transfer
+certifi==2021.10.8
+    # via
+    #   -r requirements/base.txt
+    #   requests
+cffi==1.15.0
+    # via
+    #   -r requirements/base.txt
+    #   cryptography
+charset-normalizer==2.0.9
+    # via
+    #   -r requirements/base.txt
+    #   requests
+click==8.0.3
+    # via code-annotations
+code-annotations==1.2.0
+    # via -r requirements/test.in
+coverage[toml]==6.2
+    # via pytest-cov
+cryptography==3.4.8
+    # via
+    #   -c requirements/constraints.txt
+    #   -r requirements/base.txt
+    #   pyopenssl
+idna==3.3
+    # via
+    #   -r requirements/base.txt
+    #   requests
+iniconfig==1.1.1
+    # via pytest
+jinja2==3.0.3
+    # via code-annotations
+jmespath==0.10.0
+    # via
+    #   -r requirements/base.txt
+    #   boto3
+    #   botocore
+markupsafe==2.0.1
+    # via jinja2
+packaging==21.3
+    # via pytest
+pbr==5.8.0
+    # via stevedore
+pluggy==1.0.0
+    # via pytest
+py==1.11.0
+    # via pytest
+pycparser==2.21
+    # via
+    #   -r requirements/base.txt
+    #   cffi
+pymongo==4.0.1
+    # via -r requirements/base.txt
+pymysql==1.0.2
+    # via -r requirements/base.txt
+pyopenssl==21.0.0
+    # via -r requirements/base.txt
+pyparsing==3.0.6
+    # via packaging
+pytest==6.2.5
+    # via
+    #   pytest-cov
+    #   pytest-django
+pytest-cov==3.0.0
+    # via -r requirements/test.in
+pytest-django==4.5.2
+    # via -r requirements/test.in
+python-dateutil==2.8.2
+    # via
+    #   -r requirements/base.txt
+    #   botocore
+python-gnupg==0.4.8
+    # via -r requirements/base.txt
+python-slugify==5.0.2
+    # via code-annotations
+pyyaml==6.0
+    # via
+    #   -r requirements/base.txt
+    #   code-annotations
+requests==2.26.0
+    # via -r requirements/base.txt
+s3transfer==0.5.0
+    # via
+    #   -r requirements/base.txt
+    #   boto3
+six==1.16.0
+    # via
+    #   -r requirements/base.txt
+    #   pyopenssl
+    #   python-dateutil
+stevedore==3.5.0
+    # via code-annotations
+text-unidecode==1.3
+    # via python-slugify
+toml==0.10.2
+    # via pytest
+tomli==2.0.0
+    # via coverage
+urllib3==1.26.7
+    # via
+    #   -r requirements/base.txt
+    #   botocore
+    #   requests
diff --git a/devops/requirements/travis.in b/devops/requirements/travis.in
new file mode 100644
index 000000000..f1ad3f9fa
--- /dev/null
+++ b/devops/requirements/travis.in
@@ -0,0 +1,4 @@
+# Requirements for running tests in Travis
+-c constraints.txt
+
+codecov                   # Code coverage reporting
diff --git a/devops/requirements/travis.txt b/devops/requirements/travis.txt
new file mode 100644
index 000000000..25c2d8d06
--- /dev/null
+++ b/devops/requirements/travis.txt
@@ -0,0 +1,20 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+certifi==2021.10.8
+    # via requests
+charset-normalizer==2.0.9
+    # via requests
+codecov==2.1.12
+    # via -r requirements/travis.in
+coverage==6.2
+    # via codecov
+idna==3.3
+    # via requests
+requests==2.26.0
+    # via codecov
+urllib3==1.26.7
+    # via requests
diff --git a/devops/resources/saml-ssl-expiration-check.sh b/devops/resources/saml-ssl-expiration-check.sh
index a02da8fe8..05b68a413 100644
--- a/devops/resources/saml-ssl-expiration-check.sh
+++ b/devops/resources/saml-ssl-expiration-check.sh
@@ -9,13 +9,11 @@ create_virtualenv --python=python3.8 --clear
 . "$venvpath/bin/activate"
 set -u
 
-HOME=/edx/var/jenkins
 
 env
 set -x
 
-cd $WORKSPACE/sysadmin
-pip install -r requirements/base.txt
+pip install -r ../eequirements/base.txt
 pip install awscli
 
 cd jenkins
@@ -23,4 +21,4 @@ set +x
 
 export SSL=$($SAML_SECRET | sed 's/\\"/"/g' | jq -r ".$SECRET_KEY")
 
-python saml-ssl-expiration-check.py -d $DAYS -v SSL
+python ../jenkins/saml-ssl-expiration-check.py -d $DAYS -v SSL