Add a support for Bearer token authentication

closes #4938
https://pulp.plan.io/issues/4938

.travis/install.sh

@@ -47,7 +47,6 @@ else
   TAG=$(git rev-parse --abbrev-ref HEAD | tr / _)
 fi
 
-
 PLUGIN=pulp_docker
 
 
@@ -99,8 +98,9 @@ spec:
   pulp_settings:
      content_host: $(hostname):24816
      token_server: $(hostname):24816/token
+     private_key_path: /var/lib/pulp/tmp/private.pem
+     public_key_path: /var/lib/pulp/tmp/public.pem
      token_signature_algorithm: ES256
-    
 CRYAML
 
 # Install k3s, lightweight Kubernetes

.travis/post_before_script.sh

@@ -8,3 +8,7 @@ machine 127.0.0.1
 login admin
 password password
 " > ~/.netrc
+
+$CMD_PREFIX bash -c "dnf install -y openssl"
+$CMD_PREFIX bash -c "openssl ecparam -genkey -name prime256v1 -noout -out /var/lib/pulp/tmp/private.key"
+$CMD_PREFIX bash -c "openssl ec -in /var/lib/pulp/tmp/private.key -pubout -out /var/lib/pulp/tmp/public.key"

.travis/pulp-smash-config.json

@@ -0,0 +1,22 @@
+
+{
+    "pulp": {
+        "auth": ["admin", "password"],
+        "selinux enabled": false,
+        "version": "3"
+    },
+    "hosts": [
+        {
+            "hostname": "localhost",
+            "roles": {
+                "api": {"port": 24817, "scheme": "http", "service": "nginx"},
+                "content": {"port": 24816, "scheme": "http", "service": "pulp_content_app"},
+                "token auth": {"private key": "/var/lib/pulp/tmp/private.pem", "public key": "/var/lib/pulp/tmp/public.pem"},
+                "pulp resource manager": {},
+                "pulp workers": {},
+                "redis": {},
+                "shell": {"transport": "kubectl"}
+            }
+        }
+    ]
+}

CHANGES/4938.feature

@@ -0,0 +1 @@
+Add support for pulling content using token authentication

docs/workflows/authentication.rst

@@ -0,0 +1,97 @@
+.. _authentication:
+
+Registry Token Authentication
+=============================
+
+Pulp registry supports the `token authentication <https://docs.docker.com/registry/spec/auth/token/>`_.
+This enables users to pull content with an authorized access. A token server grants access based on the
+user's privileges and current scope.
+
+The feature is enabled by default. However, it is required to define the following settings first:
+
+    - **A fully qualified domain name of a token server**. The token server is responsible for generating
+      Bearer tokens. Append the constant ``TOKEN_SERVER`` to the settings file ``pulp_docker/app/settings.py``.
+    - **A token signature algorithm**. A particular signature algorithm can be chosen only from the list of
+      `supported algorithms <https://pyjwt.readthedocs.io/en/latest/algorithms.html#digital-signature-algorithms>`_.
+      Pulp uses exclusively asymmetric cryptography to sign and validate tokens. Therefore, it is possible
+      only to choose from the algorithms, such as ES256, RS256, or PS256. Append the the constant
+      ``TOKEN_SIGNATURE_ALGORITHM`` with a selected algorithm to the settings file.
+    - **Paths to secure keys**. These keys are going to be used for a signing and validation of tokens.
+      Remember that the keys have to be specified in the **PEM format**. To generate keys, one could use
+      the openssl utility. In the following example, the utility is used to generate keys with the algorithm
+      ES256.
+
+          1. Generate a private key::
+
+              $ openssl ecparam -genkey -name prime256v1 -noout -out /tmp/private_key.pem
+
+          2. Generate a public key out of the private key::
+
+              $ openssl ec -in /tmp/private_key.pem -pubout -out /tmp/public_key.pem
+
+Below is provided and example of the settings file:
+
+.. code-block:: python
+
+    TOKEN_SERVER = "localhost:24816/token"
+    TOKEN_SIGNATURE_ALGORITHM = 'ES256'
+    PUBLIC_KEY_PATH = '/tmp/public_key.pem'
+    PRIVATE_KEY_PATH = '/tmp/private_key.pem'
+
+To learn more about Pulp settings, take a look at `Configuration
+<https://docs.pulpproject.org/en/3.0/nightly/installation/configuration.html>`_.
+
+Restart Pulp services in order to reload the updated settings. Pulp will fetch a domain for the token
+server and will initialize all handlers according to that. Check if the token authentication was
+successfully configured by initiating the following set of commands in your environment::
+
+    $ http 'http://localhost:24816/v2/'
+
+    HTTP/1.1 401 Access to the requested resource is not authorized. A provided Bearer token is invalid.
+    Content-Length: 92
+    Content-Type: text/plain; charset=utf-8
+    Date: Mon, 14 Oct 2019 16:46:48 GMT
+    Docker-Distribution-API-Version: registry/2.0
+    Server: Python/3.7 aiohttp/3.6.1
+    Www-Authenticate: Bearer realm="http://localhost:24816/token",service="localhost:24816"
+
+    401: Access to the requested resource is not authorized. A provided Bearer token is invalid.
+
+Send a request to a specified realm::
+
+    $ http 'http://localhost:24816/token?service=localhost:24816'
+
+    HTTP/1.1 200 OK
+    Content-Length: 566
+    Content-Type: application/json; charset=utf-8
+    Date: Mon, 14 Oct 2019 16:47:33 GMT
+    Server: Python/3.7 aiohttp/3.6.1
+
+    {
+        "expires_in": 300,
+        "issued_at": "2019-10-14T16:47:33.107118Z",
+        "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q"
+    }
+
+Use the generated token to access the root again::
+
+    $ http 'localhost:24816/v2/' --auth-type=jwt --auth="eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q"
+
+    HTTP/1.1 200 OK
+    Content-Length: 2
+    Content-Type: application/json; charset=utf-8
+    Date: Mon, 14 Oct 2019 16:50:26 GMT
+    Docker-Distribution-API-Version: registry/2.0
+    Server: Python/3.7 aiohttp/3.6.1
+
+    {}
+
+After performing multiple HTTP requests, the root responded with a default value ``{}``. Received
+token can be used to access all endpoints within the requested scope too.
+
+Regular container engines, like docker, or podman, can take advantage of the token authentication.
+The authentication is handled by the engines as shown before.
+
+.. code-block:: bash
+
+    podman pull localhost:24816/foo/bar

docs/workflows/index.rst

@@ -21,6 +21,12 @@ in the home directory. The ``.netrc`` should have the following configuration:
     login admin
     password admin
 
+One should observe that ``httpie`` uses the configuration retrieved from ``.netrc`` by default.
+Due to this, a custom Authorization header is always overwritten by the Basic Authorization with
+the provided login and password. In order to send HTTP requests which contain JWT Authorization
+headers, ensure yourself that the plugin `JWTAuth plugin <https://github.com/teracyhq/httpie-jwt-auth>`_
+was already installed.
+
 If you configured the ``admin`` user with a different password, adjust the configuration
 accordingly. If you prefer to specify the username and password with each request, please see
 ``httpie`` documentation on how to do that.
@@ -49,3 +55,4 @@ Container Workflows
    sync
    host
    manage-content
+   authentication

pulp_docker/app/authorization.py

@@ -0,0 +1,178 @@
+import base64
+import hashlib
+import random
+import uuid
+
+import jwt
+
+from datetime import datetime
+from aiohttp import web
+
+from django.conf import settings
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+
+from pulpcore.plugin.content import Handler
+
+TOKEN_EXPIRATION_TIME = 300
+
+KNOWN_SERVICES = [settings.CONTENT_HOST]
+ANONYMOUS_USER = ''
+EMPTY_ACCESS_SCOPE = '::'
+
+
+class AuthorizationService(Handler):
+    """
+    A class responsible for generating and managing a Bearer token.
+
+    This class represents a token server which manages and grants permissions
+    according to a user's scope.
+    """
+
+    async def generate_token(self, request):
+        """
+        Generate a Bearer token.
+
+        A signed JSON web token is generated in this method. The structure of the token is
+        adjusted according the documentation https://docs.docker.com/registry/spec/auth/jwt/.
+
+        Args:
+            request(:class:`~aiohttp.web.Request`): The request to prepare a response for.
+
+        Returns:
+            class:`aiohttp.web_response.Response`: A newly generated Bearer token.
+
+        """
+        with open(settings.PUBLIC_KEY_PATH, 'rb') as public_key:
+            kid = self.generate_kid_header(public_key.read())
+
+        current_datetime = datetime.now()
+
+        token_queries = TokenRequestQueries.init_from(request)
+        access = self.determine_access(ANONYMOUS_USER, token_queries.scope)
+        token_server = getattr(settings, 'TOKEN_SERVER', '')
+        claim_set = self._generate_claim_set(
+            access=[access],
+            audience=token_queries.service,
+            issued_at=int(current_datetime.timestamp()),
+            issuer=token_server,
+            subject=ANONYMOUS_USER
+        )
+
+        with open(settings.PRIVATE_KEY_PATH, 'rb') as private_key:
+            binary_token = jwt.encode(
+                claim_set, private_key.read(),
+                algorithm=settings.TOKEN_SIGNATURE_ALGORITHM,
+                headers={'kid': kid}
+            )
+        token = binary_token.decode('utf8')
+        current_datetime_utc = current_datetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+        return web.json_response({
+            'expires_in': TOKEN_EXPIRATION_TIME,
+            'issued_at': current_datetime_utc,
+            'token': token
+        })
+
+    def generate_kid_header(self, public_key):
+        """Generate kid header in a libtrust compatible format."""
+        decoded_key = self._convert_key_format_from_pem_to_der(public_key)
+        truncated_sha256 = hashlib.sha256(decoded_key).hexdigest()[:30].encode('utf8')
+        encoded_base32 = base64.b32encode(truncated_sha256).decode('utf8')
+        return self._split_into_encoded_groups(encoded_base32)
+
+    def _convert_key_format_from_pem_to_der(self, public_key):
+        key_in_pem_format = serialization.load_pem_public_key(public_key, default_backend())
+        key_in_der_format = key_in_pem_format.public_bytes(
+            serialization.Encoding.DER,
+            serialization.PublicFormat.SubjectPublicKeyInfo
+        )
+        return key_in_der_format
+
+    def _split_into_encoded_groups(self, encoded_base32):
+        """Split encoded and truncated base32 into 12 groups separated by ':'."""
+        kid = encoded_base32[:4]
+        for index, char in enumerate(encoded_base32[4:], start=0):
+            if index % 4 == 0:
+                kid += ':' + char
+            else:
+                kid += char
+        return kid
+
+    def determine_access(self, user, scope):
+        """
+        Determine access permissions for a corresponding user.
+
+        This method determines whether the user has a valid access permission or not.
+        The determination is based on role based access control. For now, the access
+        is given out to anybody because the role based access control is not implemented
+        yet.
+
+        Args:
+            user (str): A name of the user who is trying to access a registry.
+            scope (str): A requested scope.
+
+        Returns:
+            list: An intersected set of the requested and the allowed access.
+
+        """
+        typ, name, actions = scope.split(':')
+        actions_list = actions.split(',')
+        permitted_actions = list(set(actions_list).intersection(['pull']))
+        return {'type': typ, 'name': name, 'actions': permitted_actions}
+
+    def _generate_claim_set(self, issuer, issued_at, subject, audience, access):
+        token_id = str(uuid.UUID(int=random.getrandbits(128), version=4))
+        expiration = issued_at + TOKEN_EXPIRATION_TIME
+        return {
+            'access': access,
+            'aud': audience,
+            'exp': expiration,
+            'iat': issued_at,
+            'iss': issuer,
+            'jti': token_id,
+            'nbf': issued_at,
+            'sub': subject
+        }
+
+
+class TokenRequestQueries:
+    """A data class that holds data retrieved from the request queries."""
+
+    def __init__(self, scope, service):
+        """
+        Store a scope and a service.
+
+        Args:
+            scope (str): A requested scope.
+            service (str): A service that request a Bearer token.
+
+        """
+        self.scope = scope
+        self.service = service
+
+    @classmethod
+    def init_from(cls, request):
+        """
+        Initialize the actual class with data retrieved from the request queries.
+
+        In this method, a validity and a presence of required queries (scope, service)
+        is checked as well. If the scope is not specified, the method checks if a user
+        is trying to access root endpoint only. Then, the scope is not relevant anymore
+        and initialized to empty type, name, and requested actions ('::').
+        """
+        try:
+            scope = request.query['scope']
+        except KeyError:
+            if request.match_info:
+                raise web.HTTPBadRequest(reason='A scope was not provided.')
+            else:
+                scope = EMPTY_ACCESS_SCOPE
+
+        try:
+            service = request.query['service']
+        except KeyError:
+            raise web.HTTPBadRequest(reason='A service name was not provided.')
+        if service not in KNOWN_SERVICES:
+            raise web.HTTPBadRequest(reason='A provided service is unknown.')
+
+        return cls(scope, service)

pulp_docker/app/content.py

@@ -2,6 +2,7 @@
 
 from pulpcore.content import app
 from pulp_docker.app.registry import Registry
+from pulp_docker.app.authorization import AuthorizationService
 
 registry = Registry()
 
@@ -10,3 +11,7 @@
 app.add_routes([web.get(r'/v2/{path:.+}/manifests/sha256:{digest:.+}', registry.get_by_digest)])
 app.add_routes([web.get(r'/v2/{path:.+}/manifests/{tag_name}', registry.get_tag)])
 app.add_routes([web.get(r'/v2/{path:.+}/tags/list', registry.tags_list)])
+
+authorization_service = AuthorizationService()
+
+app.add_routes([web.get('/token', authorization_service.generate_token)])

pulp_docker/app/downloaders.py

@@ -45,6 +45,7 @@ async def _run(self, handle_401=True, extra_data=None):
 
         Args:
             handle_401(bool): If true, catch 401, request a new token and retry.
+
         """
         headers = {}
         repo_name = None