LNX-329 - Initial commit for RUMv2 support

README.md

@@ -123,3 +123,22 @@ logger = logging.getLogger('django')
 
 logger.warning('Something happened')
 ```
+
+
+## **Real User Monitoring (RUM)**
+
+Real user monitoring injects a script tag containing the [RUM JS](https://stackify.com/retrace-real-user-monitoring/) that is responsible for capturing information about the http requests on the browser. This approach is manual and needs to be configured.
+
+### RUM - Setup
+
+```python
+# Configuration - Standard API
+logger = stackify.getLogger(..., rum_key="YourRumKey")
+# or Configuration - Python Logging Integration
+stackify.StackifyHandler(..., rum_key="YourRumKey")
+
+# Use this to apply on views
+import stackify.rum
+
+stackify.rum.insert_rum_script()
+```

docker/stackify-python-api-test

@@ -3,6 +3,8 @@ ARG from_version
 FROM python:${from_version}
 
 ARG version
+ARG test
+ARG test_repo
 
 RUN \
     apt-get update && \
@@ -14,4 +16,7 @@ COPY . /build/
 
 RUN cat /build/requirements.txt | xargs -n 1 pip install; exit 0
 
+ENV TEST="${test}"
+ENV TEST_REPO="${test_repo}"
+
 CMD /bin/bash -c "cd /build && source test-docker-execute.sh"

stackify/compat.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+import sys
+import types
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+
+if PY2:
+    import StringIO
+    import Queue as queue  # noqa F401
+    import urlparse  # noqa F401
+    from urllib2 import HTTPError  # noqa F401
+    from urllib import unquote as unquote_core  # noqa F401
+
+    StringIO = BytesIO = StringIO.StringIO
+
+    string_types = (basestring,)  # noqa F821
+    integer_types = (int, long)  # noqa F821
+    class_types = (type, types.ClassType)
+    text_type = unicode  # noqa F821
+    binary_type = str
+    list_type = list
+    dict_type = dict
+
+    def b(s):
+        return s
+
+    def iterkeys(d, **kwargs):
+        return d.iterkeys(**kwargs)
+
+    def iteritems(d, **kwargs):
+        return d.iteritems(**kwargs)
+
+    def iterlists(d, **kwargs):
+        return d.iterlists(**kwargs)
+
+    def unquote(*args, **kwargs): # noqa F811
+        return unquote_core(*args, **kwargs)
+else:
+    import io
+    import queue  # noqa F401
+    from urllib import parse as urlparse  # noqa F401
+    from urllib.error import HTTPError  # noqa F401
+
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+
+    string_types = (str,)
+    integer_types = (int,)
+    class_types = (type,)
+    text_type = str
+    binary_type = bytes
+    list_type = list
+    dict_type = dict
+
+    def b(s):
+        return s.encode("latin-1")
+
+    def iterkeys(d, **kwargs):
+        return iter(d.keys(**kwargs))
+
+    def iteritems(d, **kwargs):
+        return iter(d.items(**kwargs))
+
+    def iterlists(d, **kwargs):
+        return iter(d.lists(**kwargs))
+
+    def unquote(*args, **kwargs):
+        return urlparse.unquote(*args, **kwargs)
+
+
+def multidict_to_dict(d):
+    return dict((k, v[0] if len(v) == 1 else v) for k, v in iterlists(d))

stackify/config.py

@@ -0,0 +1,6 @@
+from stackify.constants import DEFAULT_RUM_KEY, DEFAULT_RUM_SCRIPT_URL
+
+rum_key = DEFAULT_RUM_KEY
+rum_script_url = DEFAULT_RUM_SCRIPT_URL
+application = None
+environment = None

stackify/constants.py

@@ -38,3 +38,6 @@
 TRANSPORT_TYPE_DEFAULT = 'default'
 TRANSPORT_TYPE_AGENT_SOCKET = 'agent_socket'
 TRANSPORT_TYPE_AGENT_HTTP = 'agent_http'
+
+DEFAULT_RUM_SCRIPT_URL = "https://stckjs.stackify.com/stckjs.js"
+DEFAULT_RUM_KEY = ""

stackify/rum.py

@@ -0,0 +1,86 @@
+import json
+import base64
+from stackify import config
+
+apm_installed = False
+
+try:
+    from stackifyapm import insert_rum_script as insert_rum_script_from_apm
+    apm_installed = True
+except (ImportError):
+    pass
+
+
+def insert_rum_script():
+    apm_rum_script = insert_rum_script_apm()
+    if apm_rum_script is not None:
+        return apm_rum_script
+
+    rum_key = config.rum_key
+    rum_script_url = config.rum_script_url
+
+    if not rum_script_url or not rum_key:
+        return ''
+
+    transaction_id = get_transaction_id()
+    if not transaction_id:
+        return ''
+
+    reporting_url = get_reporting_url()
+    if not reporting_url:
+        return ''
+
+    application_name = config.application
+    if not application_name:
+        return ''
+
+    environment = config.environment
+    if not environment:
+        return ''
+
+    settings = {
+        "ID": transaction_id
+    }
+
+    if application_name:
+        application_name_b64 = base64.b64encode(application_name.encode("utf-8")).decode("utf-8")
+        if (application_name_b64):
+            settings["Name"] = application_name_b64
+
+    if environment:
+        environment_b64 = base64.b64encode(environment.encode("utf-8")).decode("utf-8")
+        if (environment_b64):
+            settings["Env"] = environment_b64
+
+    if reporting_url:
+        reporting_url_b64 = base64.b64encode(reporting_url.encode("utf-8")).decode("utf-8")
+        if (reporting_url_b64):
+            settings["Trans"] = reporting_url_b64
+
+    if not settings:
+        return ''
+
+    return '<script type="text/javascript">(window.StackifySettings || (window.StackifySettings = {}))</script><script src="{}" data-key="{}" async></script>'.format(
+        json.dumps(settings),
+        rum_script_url,
+        rum_key
+    )
+
+
+def get_transaction_id():
+    return ''
+
+
+def get_reporting_url():
+    return ''
+
+
+def insert_rum_script_apm():
+    if not is_apm_installed():
+        return None
+
+    return insert_rum_script_from_apm()
+
+
+def is_apm_installed():
+    return apm_installed

stackify/transport/application.py

@@ -1,5 +1,6 @@
 import socket
 import os
+import logging
 
 from stackify.utils import arg_or_env
 from stackify.constants import API_URL
@@ -9,6 +10,12 @@
 from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET
 from stackify.constants import TRANSPORT_TYPE_DEFAULT
 from stackify.transport.default.formats import JSONObject
+from stackify.constants import DEFAULT_RUM_SCRIPT_URL
+from stackify.constants import DEFAULT_RUM_KEY
+from stackify.utils import RegexValidator, ConfigError
+from stackify import config
+
+internal_logger = logging.getLogger(__name__)
 
 
 class EnvironmentDetail(JSONObject):
@@ -38,6 +45,8 @@ def __init__(
         socket_url=SOCKET_URL,
         transport=None,
         http_endpoint=DEFAULT_HTTP_ENDPOINT,
+        rum_script_url=DEFAULT_RUM_SCRIPT_URL,
+        rum_key=DEFAULT_RUM_KEY
     ):
         self.api_key = api_key
         self.api_url = api_url
@@ -47,6 +56,39 @@ def __init__(
         self.http_endpoint = http_endpoint
         self.transport = transport
 
+        self.rum_script_url = DEFAULT_RUM_SCRIPT_URL
+        self.rum_key = DEFAULT_RUM_KEY
+
+        # Rum config validation
+        if rum_script_url != DEFAULT_RUM_SCRIPT_URL:
+            self.validate(
+                RegexValidator("^((((https?|ftps?|gopher|telnet|nntp)://)|(mailto:|news:))(%[0-9A-Fa-f]{2}|[-\(\)_.!~*';/?:@&=+$,A-Za-z0-9])+)([).!';/?:,][\[:blank:|:blank:\]])?$"),
+                rum_script_url,
+                'rum_script_url'
+            )
+            config.rum_script_url = self.rum_script_url
+
+        if rum_key != DEFAULT_RUM_KEY:
+            self.validate(
+                RegexValidator("^[A-Za-z0-9_-]+$"),
+                rum_key,
+                'rum_key'
+            )
+            config.rum_key = self.rum_key
+
+        config.environment = self.environment
+        config.application = self.application
+
+    def validate(self, validator, value, key):
+        if not validator:
+            return
+
+        try:
+            value = validator(value, key)
+            setattr(self, key, str(value))
+        except ConfigError as e:
+            internal_logger.exception(str(e))
+
 
 def get_configuration(**kwargs):
     """
@@ -69,4 +111,6 @@ def get_configuration(**kwargs):
         socket_url=arg_or_env('socket_url', kwargs, SOCKET_URL),
         http_endpoint=arg_or_env('http_endpoint', kwargs, DEFAULT_HTTP_ENDPOINT, env_key='STACKIFY_TRANSPORT_HTTP_ENDPOINT'),
         transport=transport,
+        rum_script_url=arg_or_env('rum_script_url', kwargs, DEFAULT_RUM_SCRIPT_URL, env_key='RETRACE_RUM_SCRIPT_URL'),
+        rum_key=arg_or_env('rum_key', kwargs, DEFAULT_RUM_KEY, env_key='RETRACE_RUM_KEY')
     )

stackify/utils.py

@@ -1,6 +1,8 @@
 import os
 import json
 import logging
+import re
+from stackify import compat
 
 internal_logger = logging.getLogger(__name__)
 
@@ -36,3 +38,22 @@ def get_default_object(obj):
 
 def object_is_iterable(obj):
     return hasattr(obj, '__iter__') or isinstance(obj, str)
+
+
+class RegexValidator(object):
+    def __init__(self, regex, verbose_pattern=None):
+        self.regex = regex
+        self.verbose_pattern = verbose_pattern or regex
+
+    def __call__(self, value, field_name):
+        value = compat.text_type(value)
+        match = re.match(self.regex, value)
+        if match:
+            return value
+        raise ConfigError("{} does not match pattern {}".format(value, self.verbose_pattern), field_name)
+
+
+class ConfigError(ValueError):
+    def __init__(self, msg, field_name):
+        self.field_name = field_name
+        super(ValueError, self).__init__(msg)

test-docker-execute.sh

@@ -14,7 +14,16 @@ function runPyTest() {
     echo '<--------------------------------------------->'
     echo "Python Version $(python --version)"
     echo 'Running pytest...'
-    py.test
+    py.test --ignore=tests/rum
+
+    if [ "${TEST}" = 1 ]; then
+        pip install -i "${TEST_REPO}" stackify-python-apm; 
+    else
+        pip install stackify-python-apm;
+        py.test tests/rum
+    fi
+
+    pip uninstall -y stackify-python-apm
 }
 
 runFlake8

test-docker.sh

@@ -18,7 +18,7 @@ do
     fi
 
     echo "Building stackify-python-api-test-${i}..."
-    docker build --no-cache --build-arg from_version=${i} --build-arg version=${i} --file docker/stackify-python-api-test . -t stackify-python-api-test-${i}:latest
+    docker build --no-cache --build-arg from_version=${i} --build-arg version=${i} --build-arg test=${TEST} --build-arg test_repo=${TEST_REPO} --file docker/stackify-python-api-test . -t stackify-python-api-test-${i}:latest
 
     echo "Running stackify-python-api-test-${i}..."
     docker run --network="host" --name "stackify-python-api-test-${i}"  stackify-python-api-test-${i}:latest

tests/bases.py

@@ -23,6 +23,8 @@ def setUp(self):
             'STACKIFY_API_URL',
             'STACKIFY_TRANSPORT',
             'STACKIFY_TRANSPORT_HTTP_ENDPOINT',
+            'RETRACE_RUM_SCRIPT_URL',
+            'RETRACE_RUM_KEY'
         ]
         self.saved = {}
         for key in to_save: