SP2-4463 added ecs container metadata functions

.github/workflows/main.yml

@@ -33,6 +33,7 @@ jobs:
         pipenv check
     - name: Unit Tests and Coverage Report
       run: |
+        export PYTHONPATH="$PWD"
         pytest -n 6 --cov=./
     - name: Mypy
       run: |

Pipfile

@@ -5,6 +5,7 @@ name = "pypi"
 
 [packages]
 requests = "*"
+types-requests = "*"
 
 [dev-packages]
 black = "*"
@@ -18,4 +19,4 @@ pytest-xdist = "*"
 python_version = "3.8"
 
 [pipenv]
-allow_prereleases = true
+allow_prereleases = false

README.md

@@ -1,2 +1,55 @@
 # AWSPy
-Utility tools for running Python services in AWS.
+Utility tools for running Python services in AWS. Note: This package isn't designed to replace
+services such as [Boto3](https://github.com/boto/boto3) - the Python AWS SDK.
+
+# Features
+## Fargate Backed ECS
+- Tooling to extract container metadata, stats, and task information
+
+# Installation
+
+Install using Pip:
+
+```bash
+pip install awspy
+```
+
+# Usage
+
+Import the service, then run commands:
+
+```python
+from aws_py.ecs import Fargate
+
+Fargate().get_container_metadata_v4()
+```
+
+Each service is initialised in a common way. You can pass configuration options during
+initialisation (and if no options are provided then all options revert to their defaults):
+
+```python
+from aws_py.ecs import Fargate
+
+Fargate(raise_errors=False, logger=my_logger)
+```
+
+The options available for all services are:
+
+|Option|Type|Description|Default|
+|------|----|-----------|-------|
+|raise_errors|Boolean|Should exceptions bubble up?|True|
+|logger|Python logger|A Python logger instance to log information and errors to|Python logger (`logging.getLogger(__name__)`)|
+
+# Useful Links
+
+AWSPy:
+
+- [PyPi](https://pypi.org/project/awspy/)
+- [GitHub](https://github.com/ScholarPack/awspy)
+- [Releases](https://github.com/ScholarPack/awspy/releases)
+
+Useful Python AWS Packages
+
+- [Localstack](https://localstack.cloud/)
+- [Moto](https://github.com/spulec/moto)
+- [Boto3](https://github.com/boto/boto3)

aws_py/__init__.py

@@ -0,0 +1 @@
+from .awspy import AwsPy

aws_py/awspy.py

@@ -0,0 +1,18 @@
+import logging
+
+
+class AwsPy:
+    def __init__(
+        self, raise_errors: bool = True, logger: logging.Logger = None
+    ) -> None:
+        """
+
+        :param raise_errors: Bool, halt execution and raise exceptions if True
+        :param logger: Python logging instance to log to
+        """
+
+        self._raise_errors = raise_errors
+        if logger:
+            self._logger = logger
+        else:
+            self._logger = logging.getLogger(__name__)

aws_py/ecs/README.MD

@@ -0,0 +1,74 @@
+# Fargate Backed ECS
+Tooling to extract container metadata, stats, and task information.
+Note: This is NOT suitable for use with ECS tasks running on EC2 instances.
+Requires Fargate platform version 1.4.0 for some functions.
+
+# Usage
+
+```python
+from aws_py.ecs import Fargate
+
+Fargate().get_container_metadata_v4()
+```
+
+# Commands
+
+ *get_container_metadata_v4* This function introspects the AWS-injected environment variable
+ `ECS_CONTAINER_METADATA_URI_V4` and then makes a http request to the internal container endpoint.
+ It parses and then extracts the resulting data into a dictionary (more information available from 
+ (https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html)
+ [https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html]), but shown here as of
+ 15 July 2021:
+
+ ```json
+{
+    "DockerId": "cd189a933e5849daa93386466019ab50-2495160603",
+    "Name": "curl",
+    "DockerName": "curl",
+    "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+    "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
+    "Labels": {
+        "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
+        "com.amazonaws.ecs.container-name": "curl",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50",
+        "com.amazonaws.ecs.task-definition-family": "curltest",
+        "com.amazonaws.ecs.task-definition-version": "2"
+    },
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {
+        "CPU": 10,
+        "Memory": 128
+    },
+    "CreatedAt": "2020-10-08T20:09:11.44527186Z",
+    "StartedAt": "2020-10-08T20:09:11.44527186Z",
+    "Type": "NORMAL",
+    "Networks": [
+        {
+            "NetworkMode": "awsvpc",
+            "IPv4Addresses": [
+                "192.0.2.3"
+            ],
+            "AttachmentIndex": 0,
+            "MACAddress": "0a:de:f6:10:51:e5",
+            "IPv4SubnetCIDRBlock": "192.0.2.0/24",
+            "DomainNameServers": [
+                "192.0.2.2"
+            ],
+            "DomainNameSearchList": [
+                "us-west-2.compute.internal"
+            ],
+            "PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal",
+            "SubnetGatewayIpv4Address": "192.0.2.0/24"
+        }
+    ],
+    "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1",
+    "LogOptions": {
+        "awslogs-create-group": "true",
+        "awslogs-group": "/ecs/containerlogs",
+        "awslogs-region": "us-west-2",
+        "awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50"
+    },
+    "LogDriver": "awslogs"
+}
+```

aws_py/ecs/__init__.py

@@ -0,0 +1 @@
+from .fargate import Fargate

aws_py/ecs/fargate.py

@@ -0,0 +1,55 @@
+import os
+import requests
+import json
+
+from aws_py import AwsPy
+from typing import Any
+
+
+class Fargate(AwsPy):
+    """
+    Utilities for working with containers running on Fargate backed ECS
+    NOT suitable for use with ECS tasks running on EC2 instances
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.ecs_container_metadata_uri_v4: str = os.environ.get(
+            "ECS_CONTAINER_METADATA_URI_V4", ""
+        )
+
+    def get_container_metadata_v4(self) -> dict:
+        """
+        Extract the Fargate container metadata, injected into the running container by ECS.
+        Note: This returns metadata for the current running container, NOT the task
+        See https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html
+        Requires Fargate platform version 1.4.0
+        :return: ECS metadata dictionary:
+            {
+                "DockerId": "cd189a933e5849daa93386466019ab50-2495160603",
+                "Name": "curl",
+                "DockerName": "curl",
+                "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+                "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
+                ...
+            }
+        :raises: requests.exceptions.RequestException, json.decoder.JSONDecodeError, RuntimeError
+        """
+        try:
+            request = requests.get(self.ecs_container_metadata_uri_v4, timeout=3)
+            if request.status_code == 200:
+                json_metadata: str = request.json()
+                parsed_metadata: dict = json.loads(json_metadata)
+                return parsed_metadata
+            else:
+                raise RuntimeError(
+                    f"Inappropriate response from metadata endpoint: {request.text}"
+                )
+        except Exception as e:
+            if self._raise_errors:
+                raise e
+            else:
+                self._logger.error(
+                    f"Unable to read container metadata for endpoint: {self.ecs_container_metadata_uri_v4}: {e}"
+                )
+            return {}

mypy.ini

@@ -6,4 +6,4 @@ disallow_incomplete_defs = True
 check_untyped_defs = True
 disallow_untyped_decorators = True
 warn_unused_configs = True
-files = services
+files=aws_py

tests/test_services/test_ecs/test_fargate.py

@@ -0,0 +1,68 @@
+import pytest
+import json
+import requests
+
+from aws_py.ecs import Fargate
+from unittest.mock import patch
+
+
+@patch("requests.get")
+def test_get_container_metadata_v4_successful(requests_get_function):
+    requests_get_function.return_value = type(
+        "fake_request", (), {"json": lambda: json.dumps({"A": "B"}), "status_code": 200}
+    )
+    result = Fargate().get_container_metadata_v4()
+    assert result == {"A": "B"}
+
+
+@patch("requests.get")
+def test_get_container_metadata_v4_failed_bad_json_raise_errors(requests_get_function):
+    requests_get_function.return_value = type(
+        "fake_request", (), {"json": lambda: "abc", "status_code": 200}
+    )
+    with pytest.raises(json.decoder.JSONDecodeError):
+        Fargate(raise_errors=True).get_container_metadata_v4()
+
+
+@patch("requests.get")
+def test_get_container_metadata_v4_failed_bad_json_swallow_errors(
+    requests_get_function,
+):
+    requests_get_function.return_value = type(
+        "fake_request", (), {"json": lambda: "abc", "status_code": 200}
+    )
+    result = Fargate(raise_errors=False).get_container_metadata_v4()
+    assert result == {}
+
+
+@patch("requests.get")
+def test_get_container_metadata_v4_raises_invalid_status_code(requests_get_function):
+    requests_get_function.return_value = type(
+        "fake_request",
+        (),
+        {"json": lambda: "abc", "status_code": 401, "text": lambda: "abc"},
+    )
+    with pytest.raises(RuntimeError):
+        Fargate(raise_errors=True).get_container_metadata_v4()
+
+
+@pytest.mark.parametrize(
+    "exception",
+    [
+        requests.exceptions.BaseHTTPError,
+        requests.exceptions.ReadTimeout,
+        requests.exceptions.RequestException,
+        requests.exceptions.Timeout,
+        requests.exceptions.ConnectionError,
+    ],
+)
+@patch("requests.get")
+def test_get_container_metadata_v4_handles_http_exceptions(
+    requests_get_function, exception
+):
+    requests_get_function.side_effect = exception
+    with pytest.raises(exception):
+        Fargate(raise_errors=True).get_container_metadata_v4()
+
+    result_no_raise = Fargate(raise_errors=False).get_container_metadata_v4()
+    assert result_no_raise == {}