Make python idp server (#836)

containers/test-apps/saml/idpserver/.gitignore

@@ -0,0 +1,2 @@
+.cache
+__pycache__

containers/test-apps/saml/idpserver/README.md

@@ -0,0 +1,19 @@
+This is a test SAML identity provider (IdP) server used during the SAML authentication integration test in Waiter.
+
+It was built using the SimpleSAMLphp (https://simplesamlphp.org/) server as a reference for the SAML assertion message format.
+
+## Building
+
+pip install dependencies:
+
+```bash
+$ pip3 install -r requirements.txt
+```
+
+## Running
+
+Run ```bin/run-idp-server```; see file for usage details.
+
+```bash
+$ ./bin/run-idp-server -p 8000 https://127.0.0.1/waiter-auth/saml/acs http://127.0.0.1:9091/waiter-auth/saml/acs myusername
+```

containers/test-apps/saml/idpserver/bin/run-idp-server

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Usage: bin/run-idp-server port expected_acs_endpoint acs_redirect_url auth_user
+#   port - port on which to listen for SAML authentication requests
+#   expected_acs_endpoint - expected AssertionConsumerServiceURL in the incoming SAML request
+#   acs_redirect_url - AssertionConsumerServiceURL to actually redirect to
+#   auth_user - the user to authenticate as
+#
+# Examples:
+#   bin/run-idp-server 8000 https://127.0.0.1/waiter-auth/saml/acs http://127.0.0.1:9091/waiter-auth/saml/acs myusername
+#
+# Run a dummy SAML identity provider (IdP) server
+# SAML authentication request can be routed to: http://localhost:<port>/
+
+set -ex
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd ${DIR}/..
+python3 python/idp_server.py "$@"

containers/test-apps/saml/idpserver/python/idp_server.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+#
+#  Copyright (c) 2019 Two Sigma Open Source, LLC
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to
+#  deal in the Software without restriction, including without limitation the
+#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+#  sell copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+#  IN THE SOFTWARE.
+#
+
+import base64
+import html
+import re
+import sys
+import zlib
+from datetime import datetime, timedelta
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from os import path
+from urllib.parse import parse_qs
+
+from lxml import etree
+from signxml import XMLSigner
+
+port = sys.argv[1]
+expected_acs_endpoint = sys.argv[2]
+acs_redirect_url = sys.argv[3]
+auth_user = sys.argv[4]
+
+idpserver_root_dir = path.join(path.dirname(path.abspath(__file__)), "..")
+
+
+def readfile(file):
+    return open(path.join(idpserver_root_dir, file)).read()
+
+
+saml_response_redirect_template = readfile("resources/saml-response-redirect-template.html")
+
+
+def format_time(time):
+    return time.replace(microsecond=0).isoformat() + 'Z'
+
+
+def make_saml_response():
+    key = readfile("resources/privatekey.pem").encode('ascii')
+    cert = readfile("resources/idp.crt").encode('ascii')
+    saml_response_template = readfile("resources/saml-response-template.xml")
+    saml_response = saml_response_template \
+        .replace("issue-instant-field", format_time(datetime.utcnow())) \
+        .replace("session-not-on-or-after-field", format_time(datetime.utcnow() + timedelta(days=1))) \
+        .replace("not-on-or-after-field", format_time(datetime.utcnow() + timedelta(minutes=5))) \
+        .replace("auth-user-field", auth_user)
+    root = etree.fromstring(saml_response)
+    signed_root = XMLSigner().sign(root, key=key, cert=cert)
+    return base64.b64encode(etree.tostring(signed_root)).decode()
+
+
+class MyHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        """Respond to a GET request."""
+        if self.path == "/healthcheck":
+            self.send_response(200)
+            self.send_header("content-type", "text/html")
+            self.end_headers()
+            self.wfile.write(b"OK")
+            return
+
+        url_tokens = self.path.split("?")
+        if not url_tokens or len(url_tokens) < 2:
+            return
+        query_params = parse_qs(url_tokens[1])
+        saml_request = query_params["SAMLRequest"][0]
+        relay_state = query_params["RelayState"][0]
+
+        saml_request_b64_decoded = base64.b64decode(saml_request)
+        saml_request_zlib_decoded = zlib.decompress(saml_request_b64_decoded, -15)
+
+        acs_endpoint_match = re.search('AssertionConsumerServiceURL="([^"]+)"', str(saml_request_zlib_decoded))
+        if acs_endpoint_match and acs_endpoint_match.group(1) == expected_acs_endpoint:
+            self.send_response(200)
+            self.send_header("content-type", "text/html")
+            self.end_headers()
+            response = saml_response_redirect_template \
+                .replace("form-action-field", html.escape(acs_redirect_url)) \
+                .replace("saml-response-field", html.escape(make_saml_response())) \
+                .replace("relay-state-field", html.escape(relay_state))
+            self.wfile.write(response.encode('ascii'))
+        else:
+            self.send_response(400)
+            self.send_header("content-type", "text/html")
+            self.end_headers()
+            self.wfile.write(b"Invalid AssertionConsumerServiceURL is SAML request. Expecting %s. SAML request: %s"
+                             % (expected_acs_endpoint.encode('ascii'), saml_request_zlib_decoded))
+        return
+
+
+def run(server_class=HTTPServer, handler_class=MyHandler):
+    server_address = ('', int(port))
+    httpd = server_class(server_address, handler_class)
+    httpd.serve_forever()
+
+
+run()

containers/test-apps/saml/idpserver/requirements.txt

@@ -0,0 +1 @@
+signxml==2.6.0

containers/test-apps/saml/idpserver/resources/idp.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDXTCCAkWgAwIBAgIJALmVVuDWu4NYMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMTYxMjMxMTQzNDQ3WhcNNDgwNjI1MTQzNDQ3WjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAzUCFozgNb1h1M0jzNRSCjhOBnR+uVbVpaWfXYIR+AhWDdEe5ryY+Cgav
+Og8bfLybyzFdehlYdDRgkedEB/GjG8aJw06l0qF4jDOAw0kEygWCu2mcH7XOxRt+
+YAH3TVHa/Hu1W3WjzkobqqqLQ8gkKWWM27fOgAZ6GieaJBN6VBSMMcPey3HWLBmc
++TYJmv1dbaO2jHhKh8pfKw0W12VM8P1PIO8gv4Phu/uuJYieBWKixBEyy0lHjyix
+YFCR12xdh4CA47q958ZRGnnDUGFVE1QhgRacJCOZ9bd5t9mr8KLaVBYTCJo5ERE8
+jymab5dPqe5qKfJsCZiqWglbjUo9twIDAQABo1AwTjAdBgNVHQ4EFgQUxpuwcs/C
+YQOyui+r1G+3KxBNhxkwHwYDVR0jBBgwFoAUxpuwcs/CYQOyui+r1G+3KxBNhxkw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAiWUKs/2x/viNCKi3Y6b
+lEuCtAGhzOOZ9EjrvJ8+COH3Rag3tVBWrcBZ3/uhhPq5gy9lqw4OkvEws99/5jFs
+X1FJ6MKBgqfuy7yh5s1YfM0ANHYczMmYpZeAcQf2CGAaVfwTTfSlzNLsF2lW/ly7
+yapFzlYSJLGoVE+OHEu8g5SlNACUEfkXw+5Eghh+KzlIN7R6Q7r2ixWNFBC/jWf7
+NKUfJyX8qIG5md1YUeT6GBW9Bm2/1/RiO24JTaYlfLdKK9TYb8sG5B+OLab2DImG
+99CJ25RkAcSobWNF5zD0O6lgOo3cEdB/ksCq3hmtlC/DlLZ/D8CJ+7VuZnS1rR2n
+aQ==
+-----END CERTIFICATE-----

containers/test-apps/saml/idpserver/resources/privatekey.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDNQIWjOA1vWHUz
+SPM1FIKOE4GdH65VtWlpZ9dghH4CFYN0R7mvJj4KBq86Dxt8vJvLMV16GVh0NGCR
+50QH8aMbxonDTqXSoXiMM4DDSQTKBYK7aZwftc7FG35gAfdNUdr8e7VbdaPOShuq
+qotDyCQpZYzbt86ABnoaJ5okE3pUFIwxw97LcdYsGZz5Ngma/V1to7aMeEqHyl8r
+DRbXZUzw/U8g7yC/g+G7+64liJ4FYqLEETLLSUePKLFgUJHXbF2HgIDjur3nxlEa
+ecNQYVUTVCGBFpwkI5n1t3m32avwotpUFhMImjkRETyPKZpvl0+p7mop8mwJmKpa
+CVuNSj23AgMBAAECggEABn4I/B20xxXcNzASiVZJvua9DdRHtmxTlkLznBj0x2oY
+y1/Nbs3d3oFRn5uEuhBZOTcphsgwdRSHDXZsP3gUObew+d2N/zieUIj8hLDVlvJP
+rU/s4U/l53Q0LiNByE9ThvL+zJLPCKJtd5uHZjB5fFm69+Q7gu8xg4xHIub+0pP5
+PHanmHCDrbgNN/oqlar4FZ2MXTgekW6Amyc/koE9hIn4Baa2Ke/B/AUGY4pMRLqp
+TArt+GTVeWeoFY9QACUpaHpJhGb/Piou6tlU57e42cLoki1f0+SARsBBKyXA7BB1
+1fMH10KQYFA68dTYWlKzQau/K4xaqg4FKmtwF66GQQKBgQD9OpNUS7oRxMHVJaBR
+TNWW+V1FXycqojekFpDijPb2X5CWV16oeWgaXp0nOHFdy9EWs3GtGpfZasaRVHsX
+SHtPh4Nb8JqHdGE0/CD6t0+4Dns8Bn9cSqtdQB7R3Jn7IMXi9X/U8LDKo+A18/Jq
+V8VgUngMny9YjMkQIbK8TRWkYQKBgQDPf4nxO6ju+tOHHORQty3bYDD0+OV3I0+L
+0yz0uPreryBVi9nY43KakH52D7UZEwwsBjjGXD+WH8xEsmBWsGNXJu025PvzIJoz
+lAEiXvMp/NmYp+tY4rDmO8RhyVocBqWHzh38m0IFOd4ByFD5nLEDrA3pDVo0aNgY
+n0GwRysZFwKBgQDkCj3m6ZMUsUWEty+aR0EJhmKyODBDOnY09IVhH2S/FexVFzUN
+LtfK9206hp/Awez3Ln2uT4Zzqq5K7fMzUniJdBWdVB004l8voeXpIe9OZuwfcBJ9
+gFi1zypx/uFDv421BzQpBN+QfOdKbvbdQVFjnqCxbSDr80yVlGMrI5fbwQKBgG09
+oRrepO7EIO8GN/GCruLK/ptKGkyhy3Q6xnVEmdb47hX7ncJA5IoZPmrblCVSUNsw
+n11XHabksL8OBgg9rt8oQEThQv/aDzTOW9aDlJNragejiBTwq99aYeZ1gjo1CZq4
+2jKubpCfyZC4rGDtrIfZYi1q+S2UcQhtd8DdhwQbAoGAAM4EpDA4yHB5yiek1p/o
+CbqRCta/Dx6Eyo0KlNAyPuFPAshupG4NBx7mT2ASfL+2VBHoi6mHSri+BDX5ryYF
+fMYvp7URYoq7w7qivRlvvEg5yoYrK13F2+Gj6xJ4jEN9m0KdM/g3mJGq0HBTIQrp
+Sm75WXsflOxuTn08LbgGc4s=
+-----END PRIVATE KEY-----

containers/test-apps/saml/idpserver/resources/saml-response-redirect-template.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+    <head>
+        <title>Redirecting to application...</title>
+    </head>
+    <body>
+        <form action="form-action-field" method="post">
+            <input type="hidden" name="SAMLResponse" value="saml-response-field"/>
+            <input type="hidden" name="RelayState" value="relay-state-field"/>
+            <noscript>
+                <p>JavaScript is disabled. Click "Continue" to continue to your application.</p>
+                <input type="submit" value="Continue"/>
+            </noscript>
+        </form>
+        <script language="JavaScript">window.setTimeout('document.forms[0].submit()', 0);</script>
+    </body>
+</html>

containers/test-apps/saml/idpserver/resources/saml-response-template.xml

@@ -0,0 +1,47 @@
+<samlp:Response xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol" xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+                ID="_304b128835edaa863d4b417a436287d4683466bc92" Version="2.0" IssueInstant="issue-instant-field"
+                Destination="http://localhost:9091/waiter-auth/saml/acs"
+                InResponseTo="WAITER-cd16a263-428f-4ced-818b-4c8ae39ef04d"
+                xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
+    <saml:Issuer>https://localhost:8443/simplesaml/saml2/idp/metadata.php</saml:Issuer>
+    <samlp:Status>
+        <samlp:StatusCode Value="urn:oasis:names:tc:SAML:2.0:status:Success"/>
+    </samlp:Status>
+    <saml:Assertion xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xs="http://www.w3.org/2001/XMLSchema"
+                    ID="_0961dc799196b1db16692458c81ef46b0bc1f61d40" Version="2.0" IssueInstant="issue-instant-field">
+        <saml:Issuer>https://localhost:8443/simplesaml/saml2/idp/metadata.php</saml:Issuer>
+        <ds:Signature Id="placeholder"></ds:Signature>
+        <saml:Subject>
+            <saml:NameID SPNameQualifier="waiter" Format="urn:oasis:names:tc:SAML:2.0:nameid-format:transient">
+                _c2c02940517f53c3ea1673f6406fb34fd39aa7bcf6
+            </saml:NameID>
+            <saml:SubjectConfirmation Method="urn:oasis:names:tc:SAML:2.0:cm:bearer">
+                <saml:SubjectConfirmationData NotOnOrAfter="not-on-or-after-field"
+                                              Recipient="http://localhost:9091/waiter-auth/saml/acs"
+                                              InResponseTo="WAITER-cd16a263-428f-4ced-818b-4c8ae39ef04d"/>
+            </saml:SubjectConfirmation>
+        </saml:Subject>
+        <saml:Conditions NotBefore="2019-05-15T21:47:16Z" NotOnOrAfter="not-on-or-after-field">
+            <saml:AudienceRestriction>
+                <saml:Audience>waiter</saml:Audience>
+            </saml:AudienceRestriction>
+        </saml:Conditions>
+        <saml:AuthnStatement AuthnInstant="issue-instant-field" SessionNotOnOrAfter="session-not-on-or-after-field"
+                             SessionIndex="_b85d82cd923ced243c056fb26ce08c8e3819ebf49e">
+            <saml:AuthnContext>
+                <saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:Password</saml:AuthnContextClassRef>
+            </saml:AuthnContext>
+        </saml:AuthnStatement>
+        <saml:AttributeStatement>
+            <saml:Attribute Name="uid" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
+                <saml:AttributeValue xsi:type="xs:string">1</saml:AttributeValue>
+            </saml:Attribute>
+            <saml:Attribute Name="eduPersonAffiliation" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
+                <saml:AttributeValue xsi:type="xs:string">group1</saml:AttributeValue>
+            </saml:Attribute>
+            <saml:Attribute Name="email" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
+                <saml:AttributeValue xsi:type="xs:string">auth-user-field@example.com</saml:AttributeValue>
+            </saml:Attribute>
+        </saml:AttributeStatement>
+    </saml:Assertion>
+</samlp:Response>

waiter/bin/ci/run-integration-tests-composite-scheduler.sh

@@ -15,22 +15,13 @@ TEST_COMMAND=${1:-parallel-test}
 TEST_SELECTOR=${2:-integration}
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-WAITER_DIR=${DIR}/../..
+export WAITER_DIR=${DIR}/../..
 TEST_APPS_DIR=${WAITER_DIR}/../containers/test-apps
 COURIER_DIR=${TEST_APPS_DIR}/courier
 KITCHEN_DIR=${TEST_APPS_DIR}/kitchen
 NGINX_DIR=${TEST_APPS_DIR}/nginx
 SEDIMENT_DIR=${TEST_APPS_DIR}/sediment
 
-# set SAML authenticator variables
-export SAML_IDP_URI="https://localhost:8443/simplesaml/saml2/idp/SSOService.php"
-export SAML_IDP_CERT_URI="${WAITER_DIR}/test-files/saml/idp.crt"
-export SAML_AUTH_USER="user2"
-if [[ $TEST_SELECTOR =~ fast$ ]]; then
-    # Start SAML IdP test server
-    ${DIR}/saml-idp-server-setup.sh
-fi
-
 # prepare courier server build
 pushd ${COURIER_DIR}
 mvn clean package
@@ -41,14 +32,23 @@ pushd ${SEDIMENT_DIR}
 mvn clean package
 popd
 
-# Start waiter
+# set SAML authenticator variables
+export SAML_IDP_PORT=8443
+export SAML_IDP_URI="http://localhost:${SAML_IDP_PORT}/"
+export SAML_IDP_CERT_URI="${WAITER_DIR}/test-files/saml/idp.crt"
 : ${WAITER_PORT:=9091}
+export WAITER_URI=127.0.0.1:${WAITER_PORT}
+if [[ $TEST_SELECTOR =~ fast$ ]]; then
+    # Start SAML IdP test server
+    ${DIR}/saml-idp-server-setup.sh
+fi
+
+# Start waiter
 ${WAITER_DIR}/bin/run-using-composite-scheduler.sh ${WAITER_PORT} &
 
 # Run the integration tests
 export WAITER_TEST_COURIER_CMD=${COURIER_DIR}/bin/run-courier-server.sh
 export WAITER_TEST_KITCHEN_CMD=${KITCHEN_DIR}/bin/kitchen
 export WAITER_TEST_NGINX_CMD=${NGINX_DIR}/bin/run-nginx-server.sh
 export WAITER_TEST_SEDIMENT_CMD=${SEDIMENT_DIR}/bin/run-sediment-server.sh
-export WAITER_URI=127.0.0.1:${WAITER_PORT}
 ${WAITER_DIR}/bin/test.sh ${TEST_COMMAND} ${TEST_SELECTOR}

waiter/bin/ci/saml-idp-server-setup.sh

@@ -1,28 +1,30 @@
 #!/bin/bash
-# Usage: run-unit-tests.sh
+# Usage: saml-idp-server-setup.sh
 #
 # Examples:
-#   run-unit-tests.sh
+#   saml-idp-server-setup.sh
 #
-# Run a test SAML identity provider (IdP) server via docker
-# Server UI will be accessible at: https://localhost:8443/simplesaml/module.php/core/frontpage_welcome.php
-# SAML authentication request can be routed to: https://localhost:8443/simplesaml/saml2/idp/SSOService.php
-# Further documentation can be found at: https://hub.docker.com/r/kristophjunge/test-saml-idp/
+# Run a dummy SAML identity provider (IdP) server
+# SAML authentication request can be routed to: http://localhost:<$SAML_IDP_PORT>/
 
 set -e
 
-echo Starting SAML IdP server docker container
-docker run --name=testsamlidp_idp --detach --rm -p 8090:8090 -p 8443:8443 \
--e SIMPLESAMLPHP_SP_ENTITY_ID=waiter \
--e SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE=http://localhost:9091/waiter-auth/saml/acs \
--e SIMPLESAMLPHP_SP_SINGLE_LOGOUT_SERVICE=http://localhost:9091/waiter-auth/saml/logout \
--d kristophjunge/test-saml-idp:1.15
+sudo apt-get install python3-pip
+sudo apt-get install python3-setuptools
+sudo -H pip3 install -r ${WAITER_DIR}/../containers/test-apps/saml/idpserver/requirements.txt
+
+echo Starting SAML IdP server
+${WAITER_DIR}/../containers/test-apps/saml/idpserver/bin/run-idp-server \
+    $SAML_IDP_PORT \
+    https://localhost/waiter-auth/saml/acs \
+    http://${WAITER_URI}/waiter-auth/saml/acs \
+    $(id -un) &
 
 echo -n Waiting for SAML IdP server
-while ! curl -k https://localhost:8443/simplesaml/saml2/idp/metadata.php &>/dev/null; do
+while ! curl -k http://localhost:${SAML_IDP_PORT}/healthcheck &>/dev/null; do
     echo -n .
     sleep 3
 done
 echo
 echo -n SAML IdP server started successfully
-echo
+echo