From 809ceb425a2ff54ac8dd4ec1a8bf5afc5c3d25e1 Mon Sep 17 00:00:00 2001 From: Nate Kupp Date: Tue, 23 Apr 2019 05:52:26 +0200 Subject: [PATCH] Add DataDog resource (#1282) See https://docs.datadoghq.com/api/?lang=python#overview --- .../dagster_datadog/__init__.py | 4 + .../dagster_datadog/resources.py | 58 +++++++++++ .../dagster_datadog/version.py | 1 + .../dagster_datadog_tests/__init__.py | 0 .../dagster_datadog_tests/test_resources.py | 98 +++++++++++++++++++ .../dagster_datadog_tests/test_version.py | 5 + .../dagster-datadog/dev-requirements.txt | 1 + .../dagster-datadog/requirements.txt | 1 + .../libraries/dagster-datadog/setup.py | 58 +++++++++++ .../libraries/dagster-datadog/tox.ini | 15 +++ 10 files changed, 241 insertions(+) create mode 100644 python_modules/libraries/dagster-datadog/dagster_datadog/__init__.py create mode 100644 python_modules/libraries/dagster-datadog/dagster_datadog/resources.py create mode 100644 python_modules/libraries/dagster-datadog/dagster_datadog/version.py create mode 100644 python_modules/libraries/dagster-datadog/dagster_datadog_tests/__init__.py create mode 100644 python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_resources.py create mode 100644 python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_version.py create mode 100644 python_modules/libraries/dagster-datadog/dev-requirements.txt create mode 100644 python_modules/libraries/dagster-datadog/requirements.txt create mode 100644 python_modules/libraries/dagster-datadog/setup.py create mode 100644 python_modules/libraries/dagster-datadog/tox.ini diff --git a/python_modules/libraries/dagster-datadog/dagster_datadog/__init__.py b/python_modules/libraries/dagster-datadog/dagster_datadog/__init__.py new file mode 100644 index 000000000000..0e51e39f3c60 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/dagster_datadog/__init__.py @@ -0,0 +1,4 @@ +from .version import __version__ +from .resources import datadog_resource + +__all__ = ['datadog_resource'] diff --git a/python_modules/libraries/dagster-datadog/dagster_datadog/resources.py b/python_modules/libraries/dagster-datadog/dagster_datadog/resources.py new file mode 100644 index 000000000000..f4c979b334a7 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/dagster_datadog/resources.py @@ -0,0 +1,58 @@ +from datadog import initialize, statsd, DogStatsd + +from dagster import resource, Dict, Field, String + + +class DataDogResource: + '''DataDogResource + + This resource is a thin wrapper over the dogstatsd library: + + https://datadogpy.readthedocs.io/en/latest/#datadog-dogstatsd-module + + As such, we directly mirror the public API methods of DogStatsd here; you can refer to the + DataDog documentation above for how to use this resource. + ''' + + # Mirroring levels from the dogstatsd library + OK, WARNING, CRITICAL, UNKNOWN = ( + DogStatsd.OK, + DogStatsd.WARNING, + DogStatsd.CRITICAL, + DogStatsd.UNKNOWN, + ) + + def __init__(self, api_key, app_key): + initialize(api_key=api_key, app_key=app_key) + + # Pull in methods from the dogstatsd library + for method in [ + 'event', + 'gauge', + 'increment', + 'decrement', + 'histogram', + 'distribution', + 'set', + 'service_check', + 'timed', + 'timing', + ]: + setattr(self, method, getattr(statsd, method)) + + +@resource( + config_field=Field( + Dict( + { + 'api_key': Field(String, description='Datadog API key'), + 'app_key': Field(String, description='Datadog application key'), + } + ) + ), + description='This resource is for publishing to DataDog', +) +def datadog_resource(context): + return DataDogResource( + context.resource_config.get('api_key'), context.resource_config.get('app_key') + ) diff --git a/python_modules/libraries/dagster-datadog/dagster_datadog/version.py b/python_modules/libraries/dagster-datadog/dagster_datadog/version.py new file mode 100644 index 000000000000..abeeedbf5b31 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/dagster_datadog/version.py @@ -0,0 +1 @@ +__version__ = '0.4.0' diff --git a/python_modules/libraries/dagster-datadog/dagster_datadog_tests/__init__.py b/python_modules/libraries/dagster-datadog/dagster_datadog_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_resources.py b/python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_resources.py new file mode 100644 index 000000000000..8fe18a6d77c2 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_resources.py @@ -0,0 +1,98 @@ +from dagster import execute_pipeline, solid, PipelineContextDefinition, PipelineDefinition +from dagster_datadog import datadog_resource + +try: + import unittest.mock as mock +except ImportError: + import mock + + +# To support this test, we need to do the following: +# 1. Have CircleCI publish Scala/Spark jars when that code changes +# 2. Ensure we have Spark available to CircleCI +# 3. Include example / test data in this repository +@mock.patch('datadog.statsd.timing') +@mock.patch('datadog.statsd.timed') +@mock.patch('datadog.statsd.service_check') +@mock.patch('datadog.statsd.set') +@mock.patch('datadog.statsd.distribution') +@mock.patch('datadog.statsd.histogram') +@mock.patch('datadog.statsd.decrement') +@mock.patch('datadog.statsd.increment') +@mock.patch('datadog.statsd.gauge') +@mock.patch('datadog.statsd.event') +def test_datadog_resource( + event, + gauge, + increment, + decrement, + histogram, + distribution, + statsd_set, + service_check, + timed, + timing, +): + @solid + def datadog_solid(context): + assert context.resources.datadog + + # event + context.resources.datadog.event('Man down!', 'This server needs assistance.') + event.assert_called_with('Man down!', 'This server needs assistance.') + + # gauge + context.resources.datadog.gauge('users.online', 1001, tags=["protocol:http"]) + gauge.assert_called_with('users.online', 1001, tags=["protocol:http"]) + + # increment + context.resources.datadog.increment('page.views') + increment.assert_called_with('page.views') + + # decrement + context.resources.datadog.decrement('page.views') + decrement.assert_called_with('page.views') + + context.resources.datadog.histogram('album.photo.count', 26, tags=["gender:female"]) + histogram.assert_called_with('album.photo.count', 26, tags=["gender:female"]) + + context.resources.datadog.distribution('album.photo.count', 26, tags=["color:blue"]) + distribution.assert_called_with('album.photo.count', 26, tags=["color:blue"]) + + context.resources.datadog.set('visitors.uniques', 999, tags=["browser:ie"]) + statsd_set.assert_called_with('visitors.uniques', 999, tags=["browser:ie"]) + + context.resources.datadog.service_check('svc.check_name', context.resources.datadog.WARNING) + service_check.assert_called_with('svc.check_name', context.resources.datadog.WARNING) + + context.resources.datadog.timing("query.response.time", 1234) + timing.assert_called_with("query.response.time", 1234) + + @context.resources.datadog.timed + def run_fn(): + pass + + run_fn() + timed.assert_called() + + pipeline = PipelineDefinition( + name='test_datadog_resource', + solids=[datadog_solid], + context_definitions={ + 'default': PipelineContextDefinition(resources={'datadog': datadog_resource}) + }, + ) + + result = execute_pipeline( + pipeline, + { + 'context': { + 'default': { + 'resources': { + 'datadog': {'config': {'api_key': 'NOT_USED', 'app_key': 'NOT_USED'}} + } + } + } + }, + ) + assert result.success diff --git a/python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_version.py b/python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_version.py new file mode 100644 index 000000000000..4c52c044d6f3 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_version.py @@ -0,0 +1,5 @@ +from dagster_datadog.version import __version__ + + +def test_version(): + assert __version__ diff --git a/python_modules/libraries/dagster-datadog/dev-requirements.txt b/python_modules/libraries/dagster-datadog/dev-requirements.txt new file mode 100644 index 000000000000..6eb6461c7937 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/dev-requirements.txt @@ -0,0 +1 @@ +mock==2.0.0 \ No newline at end of file diff --git a/python_modules/libraries/dagster-datadog/requirements.txt b/python_modules/libraries/dagster-datadog/requirements.txt new file mode 100644 index 000000000000..e27fdfe098d7 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/requirements.txt @@ -0,0 +1 @@ +datadog==0.28.0 \ No newline at end of file diff --git a/python_modules/libraries/dagster-datadog/setup.py b/python_modules/libraries/dagster-datadog/setup.py new file mode 100644 index 000000000000..10822fcc6428 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/setup.py @@ -0,0 +1,58 @@ +import argparse +import sys + +from setuptools import find_packages, setup + +# pylint: disable=E0401, W0611 +if sys.version_info[0] < 3: + import __builtin__ as builtins +else: + import builtins + + +def get_version(name): + version = {} + with open("dagster_datadog/version.py") as fp: + exec(fp.read(), version) # pylint: disable=W0122 + + if name == 'dagster-datadog': + return version['__version__'] + elif name == 'dagster-datadog-nightly': + return version['__nightly__'] + else: + raise Exception('Shouldn\'t be here: bad package name {name}'.format(name=name)) + + +parser = argparse.ArgumentParser() +parser.add_argument('--nightly', action='store_true') + + +def _do_setup(name='dagster-datadog'): + setup( + name='dagster_datadog', + version=get_version(name), + author='Elementl', + license='Apache-2.0', + description='Package for datadog Dagster framework components.', + url='https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-datadog', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + ], + packages=find_packages(exclude=['test']), + install_requires=['dagster', 'datadog==0.28.*'], + tests_require=['mock==2.0.*'], + zip_safe=False, + ) + + +if __name__ == '__main__': + parsed, unparsed = parser.parse_known_args() + sys.argv = [sys.argv[0]] + unparsed + if parsed.nightly: + _do_setup('dagster-datadog-nightly') + else: + _do_setup('dagster-datadog') diff --git a/python_modules/libraries/dagster-datadog/tox.ini b/python_modules/libraries/dagster-datadog/tox.ini new file mode 100644 index 000000000000..7e31e2b02b98 --- /dev/null +++ b/python_modules/libraries/dagster-datadog/tox.ini @@ -0,0 +1,15 @@ +[tox] +envlist = py37,py36,py35,py27 + +[testenv] +passenv = CIRCLECI CIRCLE_* CI_PULL_REQUEST COVERALLS_REPO_TOKEN +deps = + -e ../../dagster + -r ../../dagster/dev-requirements.txt + -e . +commands = + coverage erase + pytest -vv --junitxml=test_results.xml --cov=dagster_datadog --cov-append --cov-report= + coverage report --omit='.tox/*,**/test_*.py' --skip-covered + coverage html --omit='.tox/*,**/test_*.py' + coverage xml --omit='.tox/*,**/test_*.py'