[oracle] adding integration

.travis.yml

@@ -1,28 +1,22 @@
 sudo: required
-
 language: python
 cache:
   directories:
-    - $HOME/.cache/pip
-    - $HOME/dd-agent
+    - "$HOME/.cache/pip"
+    - "$HOME/.cache/oracle"
+    - "$HOME/dd-agent"
     - vendor/cache
-
 python:
-  - "2.7"
-
+  - '2.7'
 git:
   depth: 3
-
 branches:
   only:
     - master
-
 services:
   - docker
-
 matrix:
   fast_finish: true
-
 env:
   global:
     - NOSE_FILTER="not windows and not fixme"
@@ -36,6 +30,10 @@ env:
     - EXTRAS_BRANCH=master
     - JMXFETCH_URL="https://dd-jmxfetch.s3.amazonaws.com"
     - REQ_LOCALS="$TRAVIS_BUILD_DIR,$HOME/dd-agent,$HOME/integrations-extras"
+    - ORACLE_DIR="$HOME/.cache/oracle"
+    - ORACLE_HOME="$ORACLE_DIR/instantclient"
+    - secure: B6zL+2Zi0sjcPPSU2G0J/wxHxuQ89gA8UlrhINI13bZ6euoeMiZCr0Bhh/eXlqjezj8uawaN3ux/dQNG5/u3zMk4/HHESlaPvoxG5Gocw0nh5mhqiAuYabNDOL3YWCVw53WXpDD0f64jg6rl6be4m3gU6HYiZteKFNkC00Kndg3nL+CTSvhJVVYeLSu4R6GKdPBsKwZVSibHFg1oRnBCFHzfHoQwTxPtin6AjmKu750h0hl29sfP/vC7CErVkHtyANT/L5p6Venvz7x6gLMnx6+7vEQCAHRO1OAfzInYLHj1i8vRvKgH7QjvAJVPgToLcwxXujE5ZAEhmqdnfRf23/+7/I9Ne+l9SgOh4uRVPVe8z0Jb9fJRsw508zj1z502DBvH07Dy52JSLF9xK2zQ/5SzAUoeUre1z33WLE82KcpHGg4QJme4zXRXxVNtmdWSIa8+UT/K2M1CK30kzxHr7b7tzTK+7QBNx2mZNZUQLcHvlMTi+NGJmP212ogn+zMJ827bBZvNzLq7Rs345RAPeXja+2znfLIiguRG4wrP804hvh9fUJY0psJc85iurSW8JcGHHJwy+IUGOJYEEf55EZMaoECNiq0RU+EdABykiJiP5bAhpF530f1KfUYFX6vXVoFM4rXxDxad39+DpbQ5XRguKk+KdmUiH5Hhb7qulzU=
+    - secure: pm42Zj8DnzUkbZ9Ne+HeyUzPaFj3nVMjZ2cVvtxfx3ERfJPVeXSjZSZSNagMORYNe3VngGARCutfoedokHqXC4b21rrsqNhy43wCwdyVHQQKroAfwUAS3A9mPny58LrG8RToklvGJPwHKE8Klwg4mAehK0j3jJ7OYh9hsszK+bgF9g3ygV0ZwNS7Jej8fSfAH5WOMsqAwn57doXZcrmvgwpDJDQfQuoJdEn4aQ0TYf9dSuebS9CuWTZEzUegrbjQU/EYuL3T3QBzHsG5C0efoHNgfebfRW5PAfGjfHgzD00IdB8LWlKJGxyIk6381rWLC7rQqlkMly22vv/hL/qnnhcJmvd/CpB3OgVerIqG6NVibhmK+66wKZd2j7kj9jkUyv/hcKsm7F31cAZjzN+nm6VQasgx3LD2ENI7nwqtzRIZrrbUvvTtfkXJsw65SRShhxsLkO+Tzn5bWKaOX5gR84JGpr1IRA3JS3Yal5ZaVsZefQ4bO5Ipkd8sTR7+ozetMTPy/an5IwoaK3fMzPOz1L6YeKZakrQVX0/BjSmaKRWV9PRK6r5dBIGozepWopkxiQpTvMuDVHdYSuKSkEVPlva4U1b+WyAoFF+nebPWJjIg8TQZOyYaO5vXYI6tn7UkknMWaOKC3H61x6WxhsmoszJ+wMOSVo4kuxBNZOwRt2k=
   matrix:
     - TRAVIS_FLAVOR=default
     - TRAVIS_FLAVOR=activemq_xml FLAVOR_VERSION=5.11.1
@@ -90,6 +88,7 @@ env:
     - TRAVIS_FLAVOR=nginx FLAVOR_VERSION=1.11.8
     - TRAVIS_FLAVOR=nginx FLAVOR_VERSION=1.10.2
     - TRAVIS_FLAVOR=openstack
+    - TRAVIS_FLAVOR=oracle FLAVOR_VERSION=latest
     - TRAVIS_FLAVOR=pgbouncer FLAVOR_VERSION=latest
     - TRAVIS_FLAVOR=php_fpm FLAVOR_VERSION=5.5
     - TRAVIS_FLAVOR=postgres FLAVOR_VERSION=latest
@@ -107,14 +106,13 @@ env:
     - TRAVIS_FLAVOR=sqlserver FLAVOR_VERSION=latest
     - TRAVIS_FLAVOR=statsd
     - TRAVIS_FLAVOR=supervisord FLAVOR_VERSION=3.3.0
-    - TRAVIS_FLAVOR=tcp_check    
+    - TRAVIS_FLAVOR=tcp_check
     - TRAVIS_FLAVOR=tokumx
     - TRAVIS_FLAVOR=tomcat FLAVOR_VERSION=latest
     - TRAVIS_FLAVOR=twemproxy FLAVOR_VERSION=latest
     - TRAVIS_FLAVOR=varnish FLAVOR_VERSION=4.1.7
     - TRAVIS_FLAVOR=varnish FLAVOR_VERSION=5.0.0
     # END OF TRAVIS MATRIX
-
 install:
   - bundle install
   - bundle package
@@ -123,19 +121,23 @@ install:
   - echo "$HOME/dd-agent/" > $(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")/datadog-agent.pth
   - if [ -e ~/dd-agent/requirements.txt ]; then pip install -r ~/dd-agent/requirements.txt; fi
   - bundle exec rake setup_agent_libs
-
+  - sudo apt-get -qqy install npm
+  - pip install pexpect==4.2.1
+  - mkdir -p $ORACLE_DIR
+  - $TRAVIS_BUILD_DIR/oracle/ci/resources/get_instantclient.py --agree=yes
+  - echo "$ORACLE_HOME" | sudo tee /etc/ld.so.conf.d/oracle_instantclient.conf
+  - sudo ldconfig
+  - if [ ! -e $ORACLE_HOME/libclntsh.so ]; then ln -s $ORACLE_HOME/libclntsh.so.12.1 $ORACLE_HOME/libclntsh.so; fi
 script:
   - bundle exec rake prep_travis_ci
   - bundle exec rake ci:run
   - bundle exec rake requirements
-
 # we should clean generated files before we save the cache
 # We don't want to save .pyc files
 # Since clobber only cleans the project directory,
 # everything outside of it should be cleaned, too, so we'll use find and -delete
 before_cache:
   - rake clobber
   - find $HOME/.cache/pip $HOME/dd-agent -name *.pyc -delete
-
 after_script:
   - if [[ $(docker ps -a -q) ]]; then docker stop $(docker ps -a -q); fi

Rakefile

@@ -8,6 +8,7 @@ unless ENV['CI']
   ENV['INTEGRATIONS_DIR'] = File.join(rakefile_dir, 'embedded')
   ENV['PIP_CACHE'] = File.join(rakefile_dir, '.cache/pip')
   ENV['VOLATILE_DIR'] = '/tmp/integration-sdk-testing'
+  ENV['ORACLE_DIR'] = "#{ENV['VOLATILE_DIR']}/oracle"
   ENV['CONCURRENCY'] = ENV['CONCURRENCY'] || '2'
   ENV['NOSE_FILTER'] = ENV['NOSE_FILTER'] || 'not windows'
   ENV['RUN_VENV'] = 'true'

oracle/CHANGELOG.md

@@ -0,0 +1,8 @@
+# CHANGELOG - oracle
+
+1.0.0 / Unreleased
+==================
+
+### Changes
+
+* [FEATURE] adds oracle integration.

oracle/README.md

@@ -0,0 +1,53 @@
+# Oracle Integration
+
+## Overview
+
+Get metrics from oracle service in real time to:
+
+* Visualize and monitor oracle states
+* Be notified about oracle failovers and events.
+
+## Installation
+
+Install the `dd-check-oracle` package manually or with your favorite configuration manager
+
+We cannot ship the oracle `instantclient` libraries with the agent or the standalone check due to licensing issues. Although the required `cx_Oracle` python library will be bundled, you will still need to install the `instantclient` for it to work (hard-requirement). The steps to do so would trypically be:
+
+```
+mkdir -p /opt/oracle/ && cd /opt/oracle/
+# Download Oracle Instant Client (example dir: /opt/oracle)
+unzip /opt/oracle/instantclient-basic-linux.x64-12.1.0.2.0.zip
+unzip /opt/oracle/instantclient-sdk-linux.x64-12.1.0.2.0.zip
+export ORACLE_HOME=/opt/oracle/instantclient/
+```
+From this point we'll need to make sure the relevant oracle libs are in the `LD_LIBRARY_PATH`:
+
+```
+if [ ! -e $ORACLE_HOME/libclntsh.so ]; then ln -s $ORACLE_HOME/libclntsh.so.12.1 $ORACLE_HOME/libclntsh.so; fi
+echo "$ORACLE_HOME" | sudo tee /etc/ld.so.conf.d/oracle_instantclient.conf
+sudo ldconfig
+```
+
+That should make the oracle `instantclient` dynamic libs be reachable in the host system `LD_LIBRARY_PATH` and the python package `cx_Oracle`.
+
+Please do not hesitate to contact support or open an issue should you encounter any problems.
+
+## Configuration
+
+Edit the `oracle.yaml` file to point to your server and port, set the masters to monitor
+
+## Validation
+
+When you run `datadog-agent info` you should see something like the following:
+
+    Checks
+    ======
+
+        oracle
+        -----------
+          - instance #0 [OK]
+          - Collected 18 metrics, 0 events & 1 service checks
+
+## Compatibility
+
+The oracle check is currently compatible with the linux and darwin-based OS

oracle/check.py

@@ -0,0 +1,124 @@
+# (C) Datadog, Inc. 2010-2017
+# All rights reserved
+# Licensed under Simplified BSD License (see LICENSE)
+
+# stdlib
+
+# 3rd party
+try:
+    import cx_Oracle
+except Exception as e:
+    cx_Oracle = None
+
+# project
+from checks import AgentCheck
+
+EVENT_TYPE = SOURCE_TYPE_NAME = 'oracle'
+
+
+class Oracle(AgentCheck):
+
+    SERVICE_CHECK_NAME = 'oracle.can_connect'
+    SYS_METRICS = {
+        'Buffer Cache Hit Ratio':           'oracle.buffer_cachehit_ratio',
+        'Cursor Cache Hit Ratio':           'oracle.cursor_cachehit_ratio',
+        'Library Cache Hit Ratio':          'oracle.library_cachehit_ratio',
+        'Shared Pool Free %':               'oracle.shared_pool_free',
+        'Physical Reads Per Sec':           'oracle.physical_reads',
+        'Physical Writes Per Sec':          'oracle.physical_writes',
+        'Enqueue Timeouts Per Sec':         'oracle.enqueue_timeouts',
+        'GC CR Block Received Per Second':  'oracle.gc_cr_receive_time',
+        'Global Cache Blocks Corrupted':    'oracle.cache_blocks_corrupt',
+        'Global Cache Blocks Lost':         'oracle.cache_blocks_lost',
+        'Logons Per Sec':                   'oracle.logons',
+        'Average Active Sessions':          'oracle.active_sessions',
+        'Long Table Scans Per Sec':         'oracle.long_table_scans',
+        'SQL Service Response Time':        'oracle.service_response_time',
+        'User Rollbacks Per Sec':           'oracle.user_rollbacks',
+        'Total Sorts Per User Call':        'oracle.sorts_per_user_call',
+        'Rows Per Sort':                    'oracle.rows_per_sort',
+        'Disk Sort Per Sec':                'oracle.disk_sorts',
+        'Memory Sorts Ratio':               'oracle.memory_sorts_ratio',
+        'Database Wait Time Ratio':         'oracle.database_wait_time_ratio',
+        'Enqueue Timeouts Per Sec':         'oracle.enqueue_timeouts',
+        'Session Limit %':                  'oracle.session_limit_usage',
+        'Session Count':                    'oracle.session_count',
+        'Temp Space Used':                  'oracle.temp_space_used',
+    }
+
+    def check(self, instance):
+        if not cx_Oracle:
+            msg = """Cannot run the Oracle check until the Oracle instant client is available:
+            http://www.oracle.com/technetwork/database/features/instant-client/index.html
+            You will also need to ensure the `LD_LIBRARY_PATH` is also updated so the libs
+            are reachable.
+            """
+            self.log.error(msg)
+            return
+
+        self.log.debug('Running cx_Oracle version {0}'.format(cx_Oracle.version))
+        server, user, password, service, tags = self._get_config(instance)
+
+        if not server or not user:
+            raise Exception("Oracle host and user are needed")
+
+        con = self._get_connection(server, user, password, service)
+
+        self._get_sys_metrics(con, tags)
+
+        self._get_tablespace_metrics(con)
+
+    def _get_config(self, instance):
+        self.server = instance.get('server', None)
+        user = instance.get('user', None)
+        password = instance.get('password', None)
+        service = instance.get('service_name', None)
+        tags = instance.get('tags', None)
+        return (self.server, user, password, service, tags)
+
+    def _get_connection(self, server, user, password, service):
+        self.service_check_tags = [
+            'server:%s' % server
+        ]
+        connect_string = '{0}/{1}@//{2}/{3}'.format(user, password, server, service)
+        try:
+            con = cx_Oracle.connect(connect_string)
+            self.log.debug("Connected to Oracle DB")
+            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK,
+                               tags=self.service_check_tags)
+        except Exception, e:
+            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL,
+                               tags=self.service_check_tags)
+            self.log.error(e)
+            raise
+        return con
+
+    def _get_sys_metrics(self, con, tags):
+        query = "SELECT METRIC_NAME, VALUE, BEGIN_TIME FROM GV$SYSMETRIC " \
+            "ORDER BY BEGIN_TIME"
+        cur = con.cursor()
+        cur.execute(query)
+        for row in cur:
+            metric_name = row[0]
+            metric_value = row[1]
+            if metric_name in self.SYS_METRICS:
+                self.gauge(self.SYS_METRICS[metric_name], metric_value, tags=tags)
+
+    def _get_tablespace_metrics(self, con):
+        query = "SELECT TABLESPACE_NAME, sum(BYTES), sum(MAXBYTES) FROM sys.dba_data_files GROUP BY TABLESPACE_NAME"
+        cur = con.cursor()
+        cur.execute(query)
+        for row in cur:
+            tablespace_tag = 'tablespace:%s' % row[0]
+            used = float(row[1])
+            size = float(row[2])
+            if (used >= size):
+                in_use = 100
+            elif (used == 0) or (size == 0):
+                in_use = 0
+            else:
+                in_use = used / size * 100
+
+            self.gauge('oracle.tablespace.used', used, tags=[tablespace_tag])
+            self.gauge('oracle.tablespace.size', size, tags=[tablespace_tag])
+            self.gauge('oracle.tablespace.in_use', in_use, tags=[tablespace_tag])

oracle/ci/SQL-LICENSE.txt

@@ -0,0 +1,36 @@
+LICENSE AGREEMENT FOR CX_ORACLE
+
+Copyright 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+
+Portions Copyright 2007-2015, Anthony Tuininga. All rights reserved.
+
+Portions Copyright 2001-2007, Computronix (Canada) Ltd., Edmonton, Alberta,
+Canada. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions, and the disclaimer that follows.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions, and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the names of the copyright holders nor the names of any contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+DISCLAIMER: THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+*AS IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Computronix is a registered trademark of Computronix (Canada) Ltd.
+