Keycloak (#91)

added initial keycloak support
bird-house · Nov 14, 2019 · d5b4ac2 · d5b4ac2
1 parent 8fbe0c8
commit d5b4ac2
Show file tree

Hide file tree

Showing 21 changed files with 394 additions and 114 deletions.
diff --git a/development.ini b/development.ini
@@ -32,15 +32,19 @@ twitcher.ows_security = true
 twitcher.ows_proxy = true
 twitcher.ows_proxy_protected_path = /ows
 twitcher.oauth = true
-# available types: random_token, signed_token, custom_token
-twitcher.token.type = random_token
+# available types: random_token, signed_token, custom_token, keycloak_token
+twitcher.token.type = keycloak_token
 # run "make gencert"
 twitcher.token.keyfile = key.pem
 twitcher.token.certfile = pubkey.pem
 twitcher.token.expires_in = 3600
 twitcher.token.issuer = twitcher
 # run "make gensecret"
-twitcher.token.secret = 8dc7b9a238c54bde90f57e80b6ac8b34
+twitcher.token.secret = secret
+
+# keycloak
+keycloak.url = http://localhost:8080
+keycloak.token.secret = public_key_from_keycloak
 
 ###
 # wsgi server configuration

diff --git a/docs/diagrams/twitcher-overview.png b/docs/diagrams/twitcher-overview.png
diff --git a/docs/diagrams/twitcher-overview.xml b/docs/diagrams/twitcher-overview.xml
@@ -1 +1 @@
-<mxfile userAgent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Atom/1.38.2 Chrome/61.0.3163.100 Electron/2.0.18 Safari/537.36" version="9.1.0" editor="www.draw.io" type="device"><diagram id="6b31b2c4-f107-957b-7422-773966168f4d" name="Page-1">7Zpdc9o4FIZ/DTO7N4xsYdlcAiXZzjYTpnSa7qWwha2NsRghAuyvX8mW8Yec1AkY0kmTi+AjWZbOc3T8HpEenKz2txyvozsWkLhng2Dfg596tm1ZtiP/KMshs3geygwhp4HuVBjm9D+ijUBbtzQgm0pHwVgs6Lpq9FmSEF9UbJhztqt2W7K4+tQ1DolhmPs4Nq0PNBCRXoXtFva/CA2j/MkWGmYtC+w/hpxtE/28ng2X6U/WvML5WHqhmwgHbFcywWkPTjhjIvu02k9IrHybuy277+aZ1uO8OUlEqxv0ip5wvCX5lNOJiUPujHQ5RN1g9eB4F1FB5mvsq9adxC9tkVjFuvm4IKAuBGePZMJixtOhYOAOF6BoyX0rlz1e0jgu9VwuCfJ9ZWeJ0AFiefIaxzRM5IUvV0hk57G5ZO2FJ8IF2ZdM2gW3hK2I4AfZRbfaQMerDldrqPHsCvjOQPeJyuBd3RHrgAuPYxdOlx+035sZOAaC+4e5NMw42x9eoAFOpeHZC4hQGxqBQ7xgcCka+ebQNOwGGrYHTBroDDCgZXicBDI36EvGRcRCluB4WljHVSYl/wd4Ex13jrqYYSH9lKQWGyjrv0SIg3Yp3gomTcVDvjC2zlHW6Y0+qV+DHjTpgfSngd6zsDZsy329fJ0hBOYh0b2gdrTyzItEOYmxoE/VrHoKHjNdzYm/5VSop33BBxmDNorlNMcL9SlUn/744QCZnsFERiBdUh8L0rPlk8D9aCsiNdw36b/kT/POLjffwpMZpV0q9HxyuVRY33ywIRVaHW0+75lM+JWEdJNNswHv/Zoko9nnjOlXshEXJol8jyyWrdIoJt7yUiQd0CKNwo5IWpaB8mE2/9WkRdpBz7ZBWr1+c7lVJANgIrHsrpDYJpK7bpGgBXJa6QspkO3rIBkMrokEmkhuOkUyRC7ErZAQS4o+9xpIELgikvwd9LH13/mEnb51xqh8xBExBMN+reTyavAyvanvK1ewPx9qUBsqU7PGUGkoHNfULjoG54wO6XV++KHsfaVsteEf1doHAOWGGeFUzlHttXSEdx5UJAlG6ghGXicsIZnlhiqfptO/WtDVBdExr/wkToyBBsNhZSCIrFax+5aAc34H3C8bcAiBWmp6c8ghp57lugw6ZMiSbzsq/EhV1yAryUpld/M5lXzZi5oyqbDXqMqQtckQGEo6yLI9HumGFQ2CNNCbpFA1+CtqqB445xYusKYl7Qbh0nhwZZ/j4AoY0KZ7tUtxbJbD6uBDrlOdhVCWNNTUs9H3zxPF+Q6Ha5qdl9Q7Tee3N1nRbbb9TQ5+zPBj1t7v919Xk5+sc7s4XQk5Dqi8KItkLEsc1EWlWI2kpgNpCzVEkqF93hRKZqWYbfpJrNbfJbcLVfEGiAZcz7NxqnDcC5Yn0Pyy4IEsVBKWr1u10z82Gse7Ihr3nFLtnWuuRgHVitSpmgqerXJsGKq7yhGaZ9s8PdVWmqrh5aq+a1Ffej+ShtfzOxVbZ9jAqJZbLcfcwW5XCmpoICJ7KXPV10V1AlIORx+RD6wLkwY+XSncfHP+roVPrYWvkLchrNfCMD9FeW0tDG2zFm73CpCuwYdSt7XqsHlh0uaT9KSLkM3GbPlakJfF//Zk3Yt/oILT/wE=</diagram></mxfile>
+<mxfile userAgent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Atom/1.41.0 Chrome/69.0.3497.128 Electron/4.2.7 Safari/537.36" version="9.1.0" editor="www.draw.io" type="device"><diagram id="6b31b2c4-f107-957b-7422-773966168f4d" name="Page-1">7VpRk6I4EP41Vu29WEAg4qO6M3tXt1trnVs1u48RIuQGiRfiqPfrL4GgQHBkFHTmZpyHIZ0mJP11N18n9MBkuf3C0Cr8Rn0c9SzD3/bA555lmabliH9SssskA+hmgoARXykdBDPyL1ZCQ0nXxMdJSZFTGnGyKgs9GsfY4yUZYoxuymoLGpWfukIB1gQzD0W69IH4PMykrjU4yH/HJAjzJ5twmPXMkfcYMLqO1fN6Flikv6x7ifKx1EKTEPl0UxCBux6YMEp5drXcTnAkbZubLbvv/kjvft4Mx7zRDWpFTyha43zK6cT4LjdGuhwsbzB7YLwJCcezFfJk70bAL2QhX0aqe78gQzY4o494QiPK0qGAPxjOjUNPblux7PGCRFFBc7HA0POknMZcOYjpijaKSBCLhidWiIXyWF+yssITZhxvCyJlgi+YLjFnO6Giei1D+atyV3Oo4NkcwHdspRMWgR8oRaQcLtiPfTC6uFB2r8fA0SD4/jATgimj290zaBiXouFacwBhEzR8B7u+fS008uBQaFg1aFjA0NGALYABTM3i2Be5QTUp4yENaIyiu4N0XMakYH8fJeE+cmRjiriwU5xKLENK/8ac75RJ0ZpTITo85CulqxzKKnqjz/JPQw/o6Bnprwa9o2AldM08tXyVIThiAVZaufGlZZ5FlOEIcfJUzqqXwKOnqxn21oxw+bSvaCd80IKRmOZ4Lq8CefXp+2jNQ3nbD2Gn+Dddo8sgm7siczRLea6Hr5fyqkEGrhhk7pGM9xcOSJJNsw7GFY5H0z961iRVTfiVkYSei+eLRukSYXdxLSQd+3S63L/Q2kbSNDUoH6azt0YhUgU12xoK9fLgGpQhsY0aSKyuILF0SL51CwmcQ6cRjxBE2LoNJLZ9S0iADsl9p5AM4QCgRpBgU5C7wS0ggcYNIcnfQe+b57VH4NStU0rEI/YQA2PYr5RWbgW8jFeq+4qV6umh7MpQGWvVhkpdYb+mZt5ht+kdwups91PK+5LBKsEv2ds3DJgLppgRMUcZa+kIr9ypcOyP5FaLaMc0xpnknkibptO/mdM5x/LKCT/RBrKHw9JAAJqNfPcch3M+HO7NOhyERiU1ne1y0KlmuS6dDmq05MeGcC+UVbSRlWSF8rp+P0q87HmFmZSwV1AVQVYijWBI6kA8FI1Ux5L4furodVSo7PwlNlR1nLaJC6hWXDXExXJriIvVxgaVoYF2t5VRiiK9HJYbH2Kdwqic0Limpv4T77yIosesqO73+y+rqS/mqV3sjgQM+UQ0iiQXiRIFdlHplT2hbuPYhDWeoHGXs1xBr/SyoJ1Ecv1d4vYmqnCnDM7giuUF0Df1C6n1A5+Up7k3xGfQJt965cTpdsQItFb+1QzVXfkH9A1qlm5Np8cLaClDMZ4nq9S2iiglmD0REaunX5+vgzG1EMCwkmBNR4/gQVc0aFgD0T9rnHAdAZF4Q6FKRdzJ/1yeAr0nnECVpdTg1BVdzYP0o7B9g4UtANXCFuRbIi8tbIGlF7bNXgXCNGhXUFtJheSZSetPUpM+uGw25rmvB1s/9BqjhHhCdKrgeobzveKTZi3V1Hhj8/PJmpPmzs4n7ZqPa0bZa+B/Q8IvQse+HQe3398RTCVhNwLqwhxe2cM+m4BXx+mOfds6tZNfHh0lbp88FcmGx7Avsy+KEunDlD0uIrppsqX1mrjeRfFsmqe5nml0RPacdsnelvCM6zmq9Uv1NA3kpkFX/uAt97/iF2+ZrP0P3ETz8LFvFiuHL6rB3X8=</diagram></mxfile>
diff --git a/docs/source/_images/keycloak-client-secret.png b/docs/source/_images/keycloak-client-secret.png
diff --git a/docs/source/_images/keycloak-realm-public-key.png b/docs/source/_images/keycloak-realm-public-key.png
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -29,6 +29,7 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'nbsphinx',
     'sphinx.ext.autodoc',
     'sphinx.ext.todo',
     'sphinx.ext.viewcode',
@@ -288,3 +289,7 @@
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+
+# -- nbsphinx options ----------------------------------------------------------
+# https://nbsphinx.readthedocs.io/en/0.4.2/executing-notebooks.html
+nbsphinx_execute = 'never'
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
@@ -21,8 +21,8 @@ The URL of the Twitcher service endpoint:
 Basic Authentication
 --------------------
 
-Twitcher uses basic authentication for client application registration.
-Edit username in password in the configuration:
+Twitcher uses basic authentication for client application registration and service registration.
+Edit username and password in the configuration:
 
 .. code-block:: ini
 
@@ -33,8 +33,8 @@ Edit username in password in the configuration:
 OAuth2 Token Generator
 ----------------------
 
-Twitcher uses `OAuth2 tokens`_ to control access to the service registration and the OWS service access.
-You can use three types of tokens.
+Twitcher uses `OAuth2 tokens`_ to control OWS service access.
+You can use several types of tokens.
 
 Random Token
 ++++++++++++
@@ -82,5 +82,25 @@ Edit the configuration file:
   twitcher.token.type = custom_token
   twitcher.token.secret = secret
 
+Keycloak Token
+++++++++++++++
+
+JWT tokens generated by a Keycloak_ OAuth2 service.
+
+Edit the configuration file:
+
+.. code-block:: ini
+
+  twitcher.token.type = keycloak_token
+
+You need to copy the public key of your Keycloak realm to the configuration (see screenshot):
+
+.. code-block:: ini
+
+  keycloak.token.secret = secret
+
+.. image:: _images/keycloak-realm-public-key.png
+
 .. _OAuth2 tokens: https://oauthlib.readthedocs.io/en/latest/oauth2/tokens/bearer.html
 .. _JWT tokens: https://pyjwt.readthedocs.io/en/latest/usage.html
+.. _Keycloak: https://www.keycloak.org/
diff --git a/docs/source/notebooks b/docs/source/notebooks
@@ -0,0 +1 @@
+../../notebooks
diff --git a/docs/source/overview.rst b/docs/source/overview.rst
@@ -10,10 +10,14 @@ Twitcher Components
 Twitcher consists of the following main parts:
 
 OWS Security
-   A security layer to protect service access with x509 certificates and :ref:`oauth2_api`.
+   A security layer to protect service access with :ref:`oauth2_api`.
 OWS Registry
-   :ref:`ows_registry_api` is a registration service with an :ref:`openapi_api` to register OWS services for the OWS proxy.
+   :ref:`ows_registry_api` is a registration service with an :ref:`openapi_api` to register OWS services for the OWS proxy protected by basic authentication.
 OWS Proxy
    :ref:`ows_proxy_api` a service which acts as a proxy for registered OWS services.
 
+OAuth access tokens can be retrieved from a Keycloak authentication service using the `client credentials workflow`_.
+
 .. image:: _images/twitcher-overview.png
+
+.. _`client credentials workflow`: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#refreshing-tokens
diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst
@@ -43,7 +43,7 @@ a *client_id* and *client_secret*:
 
 .. code-block:: console
 
-  $ twitcherctl -k add --username demo --password demo --name demo_app
+  $ twitcherctl -k --username demo --password demo add --name demo_app
   {'name': 'demo_app', 'client_id': 'id', 'client_secret': 'secret'}
 
 Get an access token to use the registration service using your
@@ -57,19 +57,19 @@ OAuth *client_id* and *client_secret* with scope *register*:
 Register a WPS service
 ----------------------
 
-Register the Emu WPS service at the Twitcher ``OWSProxy`` using an OAuth access token.
+Register the Emu WPS service at the Twitcher ``OWSProxy``:
 
 .. code-block:: console
 
-   $ twitcherctl -k -t TOKEN register --name emu http://localhost:5000/wps
+   $ twitcherctl -k --username demo --password demo register --name emu http://localhost:5000/wps
 
 If you don't provide a name with ``--name`` option then a nice name will be generated, for example ``sleepy_flamingo``.
 
 Use the ``list`` command to see which WPS services are registered with OWSProxy:
 
 .. code-block:: console
 
-   $ twitcherctl -k list
+   $ twitcherctl -k --username demo --password demo list
    [{'url': 'http://localhost:5000/wps', 'type': 'wps', 'name': 'emu', 'auth': 'token'}]
 
 
@@ -147,7 +147,7 @@ Register the Emu WPS service at the Twitcher ``OWSProxy`` with ``auth`` option `
 
 .. code-block:: console
 
-   $ twitcherctl -k -t TOKEN register --name emu --auth cert http://localhost:5000/wps
+   $ twitcherctl -k --username demo --password demo register --name emu --auth cert http://localhost:5000/wps
 
 The ``GetCapabilities``  and ``DescribeProcess`` requests are not blocked:
 
@@ -171,7 +171,43 @@ Let's say your proxy certificate is ``cert.pem``, then run the exceute request a
 
   $ curl --cert cert.pem --key cert.pem -k "http://localhost:8000/ows/proxy/emu?service=WPS&version=1.0.0request=Execute&identifier=hello&DataInputs=name=tux"
 
+Keycloak example
+================
+
+Set-up a demo Keycloak service using an Ansible `playbook <https://github.com/bird-house/ansible-keycloak-playbook>`_.
+
+The keycloak service is available at (``username=admin``, ``password=admin``):
+http://localhost:8080/auth/
+
+You need to copy the public key of your Keycloak realm to the twitcher configuration (see screenshot):
+
+.. image:: _images/keycloak-realm-public-key.png
+
+Update your twitcher configuration in ``development.ini``:
+
+.. code-block:: ini
+
+  twitcher.token.type = keycloak_token
+  keycloak.token.secret = public_key_copied_from_keycloak
+
+Start the twitcher service and register the Emu_ WPS:
+
+.. code-block:: console
+
+  $ twitcherctl -k --username demo --password demo register --name emu http://localhost:5000/wps
+
+Try the demo notebook to access a token from the keycloak and execute a WPS process.
+
+Use ``client_id=demo`` and copy the client secret from Keycloak in `Clients/demo/Credentials/Secret` (see screenshot).
+
+.. image:: _images/keycloak-client-secret.png
+
+.. toctree::
+   :maxdepth: 1
+
+   notebooks/twitcher-keycloak-demo
 
 .. _ESGF: https://esgf.llnl.gov/
 .. _esgf-pyclient: https://github.com/ESGF/esgf-pyclient
 .. _playbook: https://github.com/bird-house/ansible-wps-playbook
+.. _Emu: https://github.com/bird-house/emu
diff --git a/environment-docs.yml b/environment-docs.yml
@@ -0,0 +1,13 @@
+# conda env create -f environment-docs.yml
+name: twitcher
+channels:
+  - defaults
+dependencies:
+  - pip
+  # twitcher
+  - lxml
+  - requests
+  # sphinx
+  - sphinx
+  - ipython
+  - nbsphinx
diff --git a/notebooks/environment.yml b/notebooks/environment.yml
@@ -3,4 +3,9 @@ name: twitcher
 channels:
   - defaults
 dependencies:
+  - pip
   - jupyterlab
+  - requests
+  - oauthlib
+  - pip:
+    - requests-oauthlib
diff --git a/notebooks/twitcher-client.ipynb b/notebooks/twitcher-client.ipynb
@@ -26,7 +26,7 @@
    "source": [
     "from twitcher.client import TwitcherService\n",
     "base_url = 'http://localhost:8000'\n",
-    "twitcher = TwitcherService(base_url, verify=False)"
+    "twitcher = TwitcherService(base_url, username='demo', password='demo', verify=False)"
    ]
   },
   {
@@ -42,25 +42,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "client = twitcher.add_client_app(username='demo', password='demo', name='test1', redirect_uri='http://demo/test1')\n",
-    "print(client)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Get OAuth access token to register WPS service"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "token = twitcher.fetch_token(client_id=client['client_id'], client_secret=client['client_secret'], scope='register')\n",
-    "print(token)"
+    "client = twitcher.add_client_app(name='test1', redirect_uri='http://demo/test1')\n",
+    "client"
    ]
   },
   {
@@ -76,8 +59,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "service = twitcher.register_service(access_token=token['access_token'], name=\"emu_demo\", url=\"http://localhost:5000/wps\")\n",
-    "print(service)"
+    "service = twitcher.register_service(name=\"emu_demo\", url=\"http://localhost:5000/wps\")\n",
+    "service"
    ]
   },
   {
@@ -96,7 +79,7 @@
     "import requests\n",
     "\n",
     "url = \"{}/ows/proxy/emu_demo?service=WPS&request=GetCapabilities\".format(base_url)\n",
-    "print(url)"
+    "url"
    ]
   },
   {
@@ -123,7 +106,7 @@
    "outputs": [],
    "source": [
     "url = \"{}/ows/proxy/emu_demo?service=WPS&version=1.0.0&request=DescribeProcess&identifier=hello\".format(base_url)\n",
-    "print(url)"
+    "url"
    ]
   },
   {
@@ -150,7 +133,7 @@
    "outputs": [],
    "source": [
     "url = \"{}/ows/proxy/emu_demo?service=WPS&version=1.0.0&request=Execute&identifier=hello&DataInputs=name=Tux\".format(base_url)\n",
-    "print(url)"
+    "url"
    ]
   },
   {
@@ -193,7 +176,7 @@
    "outputs": [],
    "source": [
     "compute_token = twitcher.fetch_token(client_id=client['client_id'], client_secret=client['client_secret'], scope='compute')\n",
-    "print(compute_token)"
+    "compute_token"
    ]
   },
   {
@@ -211,7 +194,7 @@
    "source": [
     "url = \"{}/ows/proxy/emu_demo?service=WPS&version=1.0.0&request=Execute&identifier=hello&DataInputs=name=Tux&access_token={}\".format(\n",
     "        base_url, compute_token['access_token'])\n",
-    "print(url)"
+    "url"
    ]
   },
   {
@@ -246,9 +229,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from twitcher.client import get_headers\n",
-    "headers = get_headers(compute_token['access_token'])\n",
-    "print(headers)"
+    "headers = {'Authorization': 'Bearer {}'.format(compute_token['access_token'])}\n",
+    "headers"
    ]
   },
   {
@@ -289,7 +271,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "version": "3.7.4"
   }
  },
  "nbformat": 4,