Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
Add DataDog resource (#1282)
- Loading branch information
Showing
10 changed files
with
241 additions
and
0 deletions.
There are no files selected for viewing
4 changes: 4 additions & 0 deletions
4
python_modules/libraries/dagster-datadog/dagster_datadog/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
from .version import __version__ | ||
from .resources import datadog_resource | ||
|
||
__all__ = ['datadog_resource'] |
58 changes: 58 additions & 0 deletions
58
python_modules/libraries/dagster-datadog/dagster_datadog/resources.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
from datadog import initialize, statsd, DogStatsd | ||
|
||
from dagster import resource, Dict, Field, String | ||
|
||
|
||
class DataDogResource: | ||
'''DataDogResource | ||
This resource is a thin wrapper over the dogstatsd library: | ||
https://datadogpy.readthedocs.io/en/latest/#datadog-dogstatsd-module | ||
As such, we directly mirror the public API methods of DogStatsd here; you can refer to the | ||
DataDog documentation above for how to use this resource. | ||
''' | ||
|
||
# Mirroring levels from the dogstatsd library | ||
OK, WARNING, CRITICAL, UNKNOWN = ( | ||
DogStatsd.OK, | ||
DogStatsd.WARNING, | ||
DogStatsd.CRITICAL, | ||
DogStatsd.UNKNOWN, | ||
) | ||
|
||
def __init__(self, api_key, app_key): | ||
initialize(api_key=api_key, app_key=app_key) | ||
|
||
# Pull in methods from the dogstatsd library | ||
for method in [ | ||
'event', | ||
'gauge', | ||
'increment', | ||
'decrement', | ||
'histogram', | ||
'distribution', | ||
'set', | ||
'service_check', | ||
'timed', | ||
'timing', | ||
]: | ||
setattr(self, method, getattr(statsd, method)) | ||
|
||
|
||
@resource( | ||
config_field=Field( | ||
Dict( | ||
{ | ||
'api_key': Field(String, description='Datadog API key'), | ||
'app_key': Field(String, description='Datadog application key'), | ||
} | ||
) | ||
), | ||
description='This resource is for publishing to DataDog', | ||
) | ||
def datadog_resource(context): | ||
return DataDogResource( | ||
context.resource_config.get('api_key'), context.resource_config.get('app_key') | ||
) |
1 change: 1 addition & 0 deletions
1
python_modules/libraries/dagster-datadog/dagster_datadog/version.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
__version__ = '0.4.0' |
Empty file.
98 changes: 98 additions & 0 deletions
98
python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_resources.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
from dagster import execute_pipeline, solid, PipelineContextDefinition, PipelineDefinition | ||
from dagster_datadog import datadog_resource | ||
|
||
try: | ||
import unittest.mock as mock | ||
except ImportError: | ||
import mock | ||
|
||
|
||
# To support this test, we need to do the following: | ||
# 1. Have CircleCI publish Scala/Spark jars when that code changes | ||
# 2. Ensure we have Spark available to CircleCI | ||
# 3. Include example / test data in this repository | ||
@mock.patch('datadog.statsd.timing') | ||
@mock.patch('datadog.statsd.timed') | ||
@mock.patch('datadog.statsd.service_check') | ||
@mock.patch('datadog.statsd.set') | ||
@mock.patch('datadog.statsd.distribution') | ||
@mock.patch('datadog.statsd.histogram') | ||
@mock.patch('datadog.statsd.decrement') | ||
@mock.patch('datadog.statsd.increment') | ||
@mock.patch('datadog.statsd.gauge') | ||
@mock.patch('datadog.statsd.event') | ||
def test_datadog_resource( | ||
event, | ||
gauge, | ||
increment, | ||
decrement, | ||
histogram, | ||
distribution, | ||
statsd_set, | ||
service_check, | ||
timed, | ||
timing, | ||
): | ||
@solid | ||
def datadog_solid(context): | ||
assert context.resources.datadog | ||
|
||
# event | ||
context.resources.datadog.event('Man down!', 'This server needs assistance.') | ||
event.assert_called_with('Man down!', 'This server needs assistance.') | ||
|
||
# gauge | ||
context.resources.datadog.gauge('users.online', 1001, tags=["protocol:http"]) | ||
gauge.assert_called_with('users.online', 1001, tags=["protocol:http"]) | ||
|
||
# increment | ||
context.resources.datadog.increment('page.views') | ||
increment.assert_called_with('page.views') | ||
|
||
# decrement | ||
context.resources.datadog.decrement('page.views') | ||
decrement.assert_called_with('page.views') | ||
|
||
context.resources.datadog.histogram('album.photo.count', 26, tags=["gender:female"]) | ||
histogram.assert_called_with('album.photo.count', 26, tags=["gender:female"]) | ||
|
||
context.resources.datadog.distribution('album.photo.count', 26, tags=["color:blue"]) | ||
distribution.assert_called_with('album.photo.count', 26, tags=["color:blue"]) | ||
|
||
context.resources.datadog.set('visitors.uniques', 999, tags=["browser:ie"]) | ||
statsd_set.assert_called_with('visitors.uniques', 999, tags=["browser:ie"]) | ||
|
||
context.resources.datadog.service_check('svc.check_name', context.resources.datadog.WARNING) | ||
service_check.assert_called_with('svc.check_name', context.resources.datadog.WARNING) | ||
|
||
context.resources.datadog.timing("query.response.time", 1234) | ||
timing.assert_called_with("query.response.time", 1234) | ||
|
||
@context.resources.datadog.timed | ||
def run_fn(): | ||
pass | ||
|
||
run_fn() | ||
timed.assert_called() | ||
|
||
pipeline = PipelineDefinition( | ||
name='test_datadog_resource', | ||
solids=[datadog_solid], | ||
context_definitions={ | ||
'default': PipelineContextDefinition(resources={'datadog': datadog_resource}) | ||
}, | ||
) | ||
|
||
result = execute_pipeline( | ||
pipeline, | ||
{ | ||
'context': { | ||
'default': { | ||
'resources': { | ||
'datadog': {'config': {'api_key': 'NOT_USED', 'app_key': 'NOT_USED'}} | ||
} | ||
} | ||
} | ||
}, | ||
) | ||
assert result.success |
5 changes: 5 additions & 0 deletions
5
python_modules/libraries/dagster-datadog/dagster_datadog_tests/test_version.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
from dagster_datadog.version import __version__ | ||
|
||
|
||
def test_version(): | ||
assert __version__ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
mock==2.0.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
datadog==0.28.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
import argparse | ||
import sys | ||
|
||
from setuptools import find_packages, setup | ||
|
||
# pylint: disable=E0401, W0611 | ||
if sys.version_info[0] < 3: | ||
import __builtin__ as builtins | ||
else: | ||
import builtins | ||
|
||
|
||
def get_version(name): | ||
version = {} | ||
with open("dagster_datadog/version.py") as fp: | ||
exec(fp.read(), version) # pylint: disable=W0122 | ||
|
||
if name == 'dagster-datadog': | ||
return version['__version__'] | ||
elif name == 'dagster-datadog-nightly': | ||
return version['__nightly__'] | ||
else: | ||
raise Exception('Shouldn\'t be here: bad package name {name}'.format(name=name)) | ||
|
||
|
||
parser = argparse.ArgumentParser() | ||
parser.add_argument('--nightly', action='store_true') | ||
|
||
|
||
def _do_setup(name='dagster-datadog'): | ||
setup( | ||
name='dagster_datadog', | ||
version=get_version(name), | ||
author='Elementl', | ||
license='Apache-2.0', | ||
description='Package for datadog Dagster framework components.', | ||
url='https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-datadog', | ||
classifiers=[ | ||
'Programming Language :: Python :: 2.7', | ||
'Programming Language :: Python :: 3.5', | ||
'Programming Language :: Python :: 3.6', | ||
'License :: OSI Approved :: Apache Software License', | ||
'Operating System :: OS Independent', | ||
], | ||
packages=find_packages(exclude=['test']), | ||
install_requires=['dagster', 'datadog==0.28.*'], | ||
tests_require=['mock==2.0.*'], | ||
zip_safe=False, | ||
) | ||
|
||
|
||
if __name__ == '__main__': | ||
parsed, unparsed = parser.parse_known_args() | ||
sys.argv = [sys.argv[0]] + unparsed | ||
if parsed.nightly: | ||
_do_setup('dagster-datadog-nightly') | ||
else: | ||
_do_setup('dagster-datadog') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
[tox] | ||
envlist = py37,py36,py35,py27 | ||
|
||
[testenv] | ||
passenv = CIRCLECI CIRCLE_* CI_PULL_REQUEST COVERALLS_REPO_TOKEN | ||
deps = | ||
-e ../../dagster | ||
-r ../../dagster/dev-requirements.txt | ||
-e . | ||
commands = | ||
coverage erase | ||
pytest -vv --junitxml=test_results.xml --cov=dagster_datadog --cov-append --cov-report= | ||
coverage report --omit='.tox/*,**/test_*.py' --skip-covered | ||
coverage html --omit='.tox/*,**/test_*.py' | ||
coverage xml --omit='.tox/*,**/test_*.py' |