Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Add DataDog resource (#1282)
  • Loading branch information
natekupp committed Apr 23, 2019
1 parent 98217a4 commit 809ceb4
Show file tree
Hide file tree
Showing 10 changed files with 241 additions and 0 deletions.
@@ -0,0 +1,4 @@
from .version import __version__
from .resources import datadog_resource

__all__ = ['datadog_resource']
@@ -0,0 +1,58 @@
from datadog import initialize, statsd, DogStatsd

from dagster import resource, Dict, Field, String


class DataDogResource:
'''DataDogResource
This resource is a thin wrapper over the dogstatsd library:
https://datadogpy.readthedocs.io/en/latest/#datadog-dogstatsd-module
As such, we directly mirror the public API methods of DogStatsd here; you can refer to the
DataDog documentation above for how to use this resource.
'''

# Mirroring levels from the dogstatsd library
OK, WARNING, CRITICAL, UNKNOWN = (
DogStatsd.OK,
DogStatsd.WARNING,
DogStatsd.CRITICAL,
DogStatsd.UNKNOWN,
)

def __init__(self, api_key, app_key):
initialize(api_key=api_key, app_key=app_key)

# Pull in methods from the dogstatsd library
for method in [
'event',
'gauge',
'increment',
'decrement',
'histogram',
'distribution',
'set',
'service_check',
'timed',
'timing',
]:
setattr(self, method, getattr(statsd, method))


@resource(
config_field=Field(
Dict(
{
'api_key': Field(String, description='Datadog API key'),
'app_key': Field(String, description='Datadog application key'),
}
)
),
description='This resource is for publishing to DataDog',
)
def datadog_resource(context):
return DataDogResource(
context.resource_config.get('api_key'), context.resource_config.get('app_key')
)
@@ -0,0 +1 @@
__version__ = '0.4.0'
Empty file.
@@ -0,0 +1,98 @@
from dagster import execute_pipeline, solid, PipelineContextDefinition, PipelineDefinition
from dagster_datadog import datadog_resource

try:
import unittest.mock as mock
except ImportError:
import mock


# To support this test, we need to do the following:
# 1. Have CircleCI publish Scala/Spark jars when that code changes
# 2. Ensure we have Spark available to CircleCI
# 3. Include example / test data in this repository
@mock.patch('datadog.statsd.timing')
@mock.patch('datadog.statsd.timed')
@mock.patch('datadog.statsd.service_check')
@mock.patch('datadog.statsd.set')
@mock.patch('datadog.statsd.distribution')
@mock.patch('datadog.statsd.histogram')
@mock.patch('datadog.statsd.decrement')
@mock.patch('datadog.statsd.increment')
@mock.patch('datadog.statsd.gauge')
@mock.patch('datadog.statsd.event')
def test_datadog_resource(
event,
gauge,
increment,
decrement,
histogram,
distribution,
statsd_set,
service_check,
timed,
timing,
):
@solid
def datadog_solid(context):
assert context.resources.datadog

# event
context.resources.datadog.event('Man down!', 'This server needs assistance.')
event.assert_called_with('Man down!', 'This server needs assistance.')

# gauge
context.resources.datadog.gauge('users.online', 1001, tags=["protocol:http"])
gauge.assert_called_with('users.online', 1001, tags=["protocol:http"])

# increment
context.resources.datadog.increment('page.views')
increment.assert_called_with('page.views')

# decrement
context.resources.datadog.decrement('page.views')
decrement.assert_called_with('page.views')

context.resources.datadog.histogram('album.photo.count', 26, tags=["gender:female"])
histogram.assert_called_with('album.photo.count', 26, tags=["gender:female"])

context.resources.datadog.distribution('album.photo.count', 26, tags=["color:blue"])
distribution.assert_called_with('album.photo.count', 26, tags=["color:blue"])

context.resources.datadog.set('visitors.uniques', 999, tags=["browser:ie"])
statsd_set.assert_called_with('visitors.uniques', 999, tags=["browser:ie"])

context.resources.datadog.service_check('svc.check_name', context.resources.datadog.WARNING)
service_check.assert_called_with('svc.check_name', context.resources.datadog.WARNING)

context.resources.datadog.timing("query.response.time", 1234)
timing.assert_called_with("query.response.time", 1234)

@context.resources.datadog.timed
def run_fn():
pass

run_fn()
timed.assert_called()

pipeline = PipelineDefinition(
name='test_datadog_resource',
solids=[datadog_solid],
context_definitions={
'default': PipelineContextDefinition(resources={'datadog': datadog_resource})
},
)

result = execute_pipeline(
pipeline,
{
'context': {
'default': {
'resources': {
'datadog': {'config': {'api_key': 'NOT_USED', 'app_key': 'NOT_USED'}}
}
}
}
},
)
assert result.success
@@ -0,0 +1,5 @@
from dagster_datadog.version import __version__


def test_version():
assert __version__
@@ -0,0 +1 @@
mock==2.0.0
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-datadog/requirements.txt
@@ -0,0 +1 @@
datadog==0.28.0
58 changes: 58 additions & 0 deletions python_modules/libraries/dagster-datadog/setup.py
@@ -0,0 +1,58 @@
import argparse
import sys

from setuptools import find_packages, setup

# pylint: disable=E0401, W0611
if sys.version_info[0] < 3:
import __builtin__ as builtins
else:
import builtins


def get_version(name):
version = {}
with open("dagster_datadog/version.py") as fp:
exec(fp.read(), version) # pylint: disable=W0122

if name == 'dagster-datadog':
return version['__version__']
elif name == 'dagster-datadog-nightly':
return version['__nightly__']
else:
raise Exception('Shouldn\'t be here: bad package name {name}'.format(name=name))


parser = argparse.ArgumentParser()
parser.add_argument('--nightly', action='store_true')


def _do_setup(name='dagster-datadog'):
setup(
name='dagster_datadog',
version=get_version(name),
author='Elementl',
license='Apache-2.0',
description='Package for datadog Dagster framework components.',
url='https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-datadog',
classifiers=[
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'License :: OSI Approved :: Apache Software License',
'Operating System :: OS Independent',
],
packages=find_packages(exclude=['test']),
install_requires=['dagster', 'datadog==0.28.*'],
tests_require=['mock==2.0.*'],
zip_safe=False,
)


if __name__ == '__main__':
parsed, unparsed = parser.parse_known_args()
sys.argv = [sys.argv[0]] + unparsed
if parsed.nightly:
_do_setup('dagster-datadog-nightly')
else:
_do_setup('dagster-datadog')
15 changes: 15 additions & 0 deletions python_modules/libraries/dagster-datadog/tox.ini
@@ -0,0 +1,15 @@
[tox]
envlist = py37,py36,py35,py27

[testenv]
passenv = CIRCLECI CIRCLE_* CI_PULL_REQUEST COVERALLS_REPO_TOKEN
deps =
-e ../../dagster
-r ../../dagster/dev-requirements.txt
-e .
commands =
coverage erase
pytest -vv --junitxml=test_results.xml --cov=dagster_datadog --cov-append --cov-report=
coverage report --omit='.tox/*,**/test_*.py' --skip-covered
coverage html --omit='.tox/*,**/test_*.py'
coverage xml --omit='.tox/*,**/test_*.py'

0 comments on commit 809ceb4

Please sign in to comment.