[Integrations] Add support for Logs (#490)

* [Logs] Add log-agent options to datadog.yaml

default to nil for all these options, as the log-agent itself
has sane default options when these parameters are missing

* [Integrations] Allow `logs` section to be taken into account

So we can generically add the logs section to the yaml integration files

* [Integrations] Update templates and recipes to include logs section
when configured.

See http://docs.datadoghq.com/logs/ for detailed configuration

attributes/default.rb

@@ -360,6 +360,13 @@
 default['datadog']['process_agent']['container_interval'] = nil
 default['datadog']['process_agent']['rtcontainer_interval'] = nil
 
+# Logs functionality settings
+# Set `enable_log_agent` to:
+# * `true` to explicitly enable the log agent
+# * `false` to explicitly disable it
+# Leave it to `nil` to let the agent's default behavior decide whether to run the log-agent
+default['datadog']['enable_logs_agent'] = nil
+
 # For custom gem servers on restricted networks
 # This attribute only works on Chef >= 12.3
 # Change false to the URL of your custom gem server

providers/monitor.rb

@@ -25,7 +25,8 @@ def whyrun_supported?
     variables(
       init_config: new_resource.init_config,
       instances:   new_resource.instances,
-      version:     new_resource.version
+      version:     new_resource.version,
+      logs:        new_resource.logs
     )
     cookbook new_resource.cookbook
     sensitive true if Chef::Resource.instance_methods(false).include?(:sensitive)

recipes/activemq.rb

@@ -22,4 +22,5 @@
 
 datadog_monitor 'activemq' do
   instances node['datadog']['activemq']['instances']
+  logs node['datadog']['activemq']['logs']
 end

recipes/apache.rb

@@ -19,4 +19,5 @@
 
 datadog_monitor 'apache' do
   instances node['datadog']['apache']['instances']
+  logs node['datadog']['apache']['logs']
 end

recipes/cacti.rb

@@ -40,4 +40,5 @@
 
 datadog_monitor 'cacti' do
   instances node['datadog']['cacti']['instances']
+  logs node['datadog']['cacti']['logs']
 end

recipes/cassandra.rb

@@ -38,4 +38,5 @@
 datadog_monitor 'cassandra' do
   instances node['datadog']['cassandra']['instances']
   version node['datadog']['cassandra']['version']
+  logs node['datadog']['cassandra']['logs']
 end

recipes/consul.rb

@@ -30,4 +30,5 @@
 
 datadog_monitor 'consul' do
   instances node['datadog']['consul']['instances']
+  logs node['datadog']['consul']['logs']
 end

recipes/couchbase.rb

@@ -18,4 +18,5 @@
 
 datadog_monitor 'couchbase' do
   instances node['datadog']['couchbase']['instances']
+  logs node['datadog']['couchbase']['logs']
 end

recipes/couchdb.rb

@@ -20,4 +20,5 @@
 datadog_monitor 'couch' do
   init_config nil
   instances node['datadog']['couch']['instances']
+  logs node['datadog']['couch']['logs']
 end

recipes/directory.rb

@@ -13,4 +13,5 @@
 
 datadog_monitor 'directory' do
   instances node['datadog']['directory']['instances']
+  logs node['datadog']['directory']['logs']
 end

recipes/disk.rb

@@ -32,4 +32,5 @@
 datadog_monitor 'disk' do
   instances node['datadog']['disk']['instances']
   init_config nil
+  logs node['datadog']['disk']['logs']
 end

recipes/dns_check.rb

@@ -30,4 +30,5 @@
 
 datadog_monitor 'dns_check' do
   instances node['datadog']['dns_check']['instances']
+  logs node['datadog']['dns_check']['logs']
 end

recipes/docker.rb

@@ -46,4 +46,5 @@
 datadog_monitor 'docker' do
   init_config node['datadog']['docker']['init_config']
   instances node['datadog']['docker']['instances']
+  logs node['datadog']['docker']['logs']
 end

recipes/docker_daemon.rb

@@ -76,4 +76,5 @@
 datadog_monitor 'docker_daemon' do
   init_config node['datadog']['docker_daemon']['init_config']
   instances node['datadog']['docker_daemon']['instances']
+  logs node['datadog']['docker_daemon']['logs']
 end

recipes/elasticsearch.rb

@@ -57,4 +57,5 @@
 
 datadog_monitor 'elastic' do
   instances node['datadog']['elasticsearch']['instances']
+  logs node['datadog']['elasticsearch']['logs']
 end

recipes/etcd.rb

@@ -21,4 +21,5 @@
 
 datadog_monitor 'etcd' do
   instances node['datadog']['etcd']['instances']
+  logs node['datadog']['etcd']['logs']
 end

recipes/fluentd.rb

@@ -12,4 +12,5 @@
 
 datadog_monitor 'fluentd' do
   instances node['datadog']['fluentd']['instances']
+  logs node['datadog']['fluentd']['logs']
 end

recipes/go-metro.rb

@@ -78,4 +78,5 @@
 datadog_monitor 'go-metro' do
   init_config node['datadog']['go-metro']['init_config']
   instances node['datadog']['go-metro']['instances']
+  logs node['datadog']['go-metro']['logs']
 end

recipes/go_expvar.rb

@@ -21,4 +21,5 @@
 datadog_monitor 'go_expvar' do
   init_config node['datadog']['go_expvar']['init_config']
   instances node['datadog']['go_expvar']['instances']
+  logs node['datadog']['go_expvar']['logs']
 end

recipes/gunicorn.rb

@@ -27,4 +27,5 @@
 
 datadog_monitor 'gunicorn' do
   instances node['datadog']['gunicorn']['instances']
+  logs node['datadog']['gunicorn']['logs']
 end

recipes/haproxy.rb

@@ -16,4 +16,5 @@
 
 datadog_monitor 'haproxy' do
   instances node['datadog']['haproxy']['instances']
+  logs node['datadog']['haproxy']['logs']
 end

recipes/hdfs.rb

@@ -26,4 +26,5 @@
 
 datadog_monitor 'hdfs' do
   instances node['datadog']['hdfs']['instances']
+  logs node['datadog']['hdfs']['logs']
 end

recipes/http_check.rb

@@ -21,4 +21,5 @@
 
 datadog_monitor 'http_check' do
   instances node['datadog']['http_check']['instances']
+  logs node['datadog']['http_check']['logs']
 end

recipes/iis.rb

@@ -21,4 +21,5 @@
 
 datadog_monitor 'iis' do
   instances node['datadog']['iis']['instances']
+  logs node['datadog']['iis']['logs']
 end

recipes/integrations.rb

@@ -38,6 +38,7 @@
   datadog_monitor name do
     init_config node['datadog'][name]['init_config']
     instances node['datadog'][name]['instances']
+    logs node['datadog'][name]['logs']
     use_integration_template true
   end
 end

recipes/jenkins.rb

@@ -20,4 +20,5 @@
 
 datadog_monitor 'jenkins' do
   instances node['datadog']['jenkins']['instances']
+  logs node['datadog']['jenkins']['logs']
 end

recipes/jmx.rb

@@ -19,4 +19,5 @@
 #   ]
 datadog_monitor 'jmx' do
   instances node['datadog']['jmx']['instances']
+  logs node['datadog']['jmx']['logs']
 end

recipes/kafka.rb

@@ -38,4 +38,5 @@
 datadog_monitor 'kafka' do
   instances node['datadog']['kafka']['instances']
   version node['datadog']['kafka']['version']
+  logs node['datadog']['kafka']['logs']
 end

recipes/kafka_consumer.rb

@@ -13,4 +13,5 @@
 
 datadog_monitor 'kafka_consumer' do
   instances node['datadog']['kafka_consumer']['instances']
+  logs node['datadog']['kafka_consumer']['logs']
 end

recipes/kubernetes.rb

@@ -27,4 +27,5 @@
 
 datadog_monitor 'kubernetes' do
   instances node['datadog']['kubernetes']['instances']
+  logs node['datadog']['kubernetes']['logs']
 end

recipes/kyototycoon.rb

@@ -19,4 +19,5 @@
 
 datadog_monitor 'kyototycoon' do
   instances node['datadog']['kyototycoon']['instances']
+  logs node['datadog']['kyototycoon']['logs']
 end

recipes/lighttpd.rb

@@ -2,4 +2,5 @@
 
 datadog_monitor 'lighttpd' do
   instances node['datadog']['lighttpd']['instances']
+  logs node['datadog']['lighttpd']['logs']
 end

recipes/memcache.rb

@@ -16,4 +16,5 @@
 
 datadog_monitor 'mcache' do
   instances node['datadog']['memcache']['instances']
+  logs node['datadog']['memcache']['logs']
 end

recipes/mesos.rb

@@ -36,4 +36,5 @@
 datadog_monitor 'mesos' do
   init_config node['datadog']['mesos']['init_config']
   instances node['datadog']['mesos']['instances']
+  logs node['datadog']['mesos']['logs']
 end

recipes/mongo.rb

@@ -11,4 +11,5 @@
 
 datadog_monitor 'mongo' do
   instances node['datadog']['mongo']['instances']
+  logs node['datadog']['mongo']['logs']
 end

recipes/mysql.rb

@@ -31,4 +31,5 @@
 
 datadog_monitor 'mysql' do
   instances node['datadog']['mysql']['instances']
+  logs node['datadog']['mysql']['logs']
 end

recipes/network.rb

@@ -13,4 +13,5 @@
 
 datadog_monitor 'network' do
   instances node['datadog']['network']['instances']
+  logs node['datadog']['network']['logs']
 end

recipes/nginx.rb

@@ -22,4 +22,5 @@
 
 datadog_monitor 'nginx' do
   instances node['datadog']['nginx']['instances']
+  logs node['datadog']['nginx']['logs']
 end

recipes/ntp.rb

@@ -15,4 +15,5 @@
 #   ]
 datadog_monitor 'ntp' do
   instances node['datadog']['ntp']['instances']
+  logs node['datadog']['ntp']['logs']
 end

recipes/pgbouncer.rb

@@ -36,4 +36,5 @@
 
 datadog_monitor 'pgbouncer' do
   instances node['datadog']['pgbouncer']['instances']
+  logs node['datadog']['pgbouncer']['logs']
 end

recipes/php_fpm.rb

@@ -15,4 +15,5 @@
 #   ]
 datadog_monitor 'php_fpm' do
   instances node['datadog']['php_fpm']['instances']
+  logs node['datadog']['php_fpm']['logs']
 end

recipes/postfix.rb

@@ -42,4 +42,5 @@
 
 datadog_monitor 'postfix' do
   instances postfix_instances
+  logs node['datadog']['postfix']['logs']
 end

recipes/postgres.rb

@@ -25,4 +25,5 @@
 
 datadog_monitor 'postgres' do
   instances node['datadog']['postgres']['instances']
+  logs node['datadog']['postgres']['logs']
 end

recipes/process.rb

@@ -26,4 +26,5 @@
 
 datadog_monitor 'process' do
   instances node['datadog']['process']['instances']
+  logs node['datadog']['process']['logs']
 end

recipes/rabbitmq.rb

@@ -19,4 +19,5 @@
 
 datadog_monitor 'rabbitmq' do
   instances node['datadog']['rabbitmq']['instances']
+  logs node['datadog']['rabbitmq']['logs']
 end

recipes/redisdb.rb

@@ -13,4 +13,5 @@
 #   ]
 datadog_monitor 'redisdb' do
   instances node['datadog']['redisdb']['instances']
+  logs node['datadog']['redisdb']['logs']
 end

recipes/riak.rb

@@ -10,4 +10,5 @@
 
 datadog_monitor 'riak' do
   instances node['datadog']['riak']['instances']
+  logs node['datadog']['riak']['logs']
 end

recipes/snmp.rb

@@ -112,4 +112,5 @@
 datadog_monitor 'snmp' do
   init_config node['datadog']['snmp']['init_config']
   instances node['datadog']['snmp']['instances']
+  logs node['datadog']['snmp']['logs']
 end

recipes/solr.rb

@@ -7,4 +7,5 @@
 
 datadog_monitor 'solr' do
   instances node['datadog']['solr']['instances']
+  logs node['datadog']['solr']['logs']
 end

recipes/sqlserver.rb

@@ -63,4 +63,5 @@
 datadog_monitor 'sqlserver' do
   init_config node['datadog']['sqlserver']['init_config']
   instances node['datadog']['sqlserver']['instances']
+  logs node['datadog']['sqlserver']['logs']
 end

recipes/ssh_check.rb

@@ -21,4 +21,5 @@
 
 datadog_monitor 'ssh_check' do
   instances node['datadog']['ssh_check']['instances']
+  logs node['datadog']['ssh_check']['logs']
 end

recipes/supervisord.rb

@@ -41,4 +41,5 @@
 
 datadog_monitor 'supervisord' do
   instances node['datadog']['supervisord']['instances']
+  logs node['datadog']['supervisord']['logs']
 end

recipes/tcp_check.rb

@@ -15,4 +15,5 @@
 
 datadog_monitor 'tcp_check' do
   instances node['datadog']['tcp_check']['instances']
+  logs node['datadog']['tcp_check']['logs']
 end

recipes/tokumx.rb

@@ -11,4 +11,5 @@
 
 datadog_monitor 'tokumx' do
   instances node['datadog']['tokumx']['instances']
+  logs node['datadog']['tokumx']['logs']
 end

recipes/tomcat.rb

@@ -22,4 +22,5 @@
 
 datadog_monitor 'tomcat' do
   instances node['datadog']['tomcat']['instances']
+  logs node['datadog']['tomcat']['logs']
 end

recipes/varnish.rb

@@ -23,4 +23,5 @@
 
 datadog_monitor 'varnish' do
   instances node['datadog']['varnish']['instances']
+  logs node['datadog']['varnish']['logs']
 end

recipes/win32_event_log.rb

@@ -20,4 +20,5 @@
 datadog_monitor 'win32_event_log' do
   init_config node['datadog']['win32_event_log']['init_config']
   instances node['datadog']['win32_event_log']['instances']
+  logs node['datadog']['win32_event_log']['logs']
 end

recipes/windows_service.rb

@@ -57,4 +57,5 @@
 
 datadog_monitor 'windows_service' do
   instances node['datadog']['windows_service']['instances']
+  logs node['datadog']['windows_service']['logs']
 end

recipes/zookeeper.rb

@@ -7,4 +7,5 @@
 
 datadog_monitor 'zk' do
   instances node['datadog']['zookeeper']['instances']
+  logs node['datadog']['zookeeper']['logs']
 end

resources/monitor.rb

@@ -6,10 +6,11 @@
 attribute :name, :kind_of => String, :name_attribute => true
 attribute :cookbook, :kind_of => String, :default => 'datadog'
 
-# checks have 2 sections: init_config and instances
+# checks have 3 sections: init_config, instances, logs
 # we mimic these here, no validation is performed until the template
 # is evaluated.
 attribute :init_config, :kind_of => Hash, :required => false, :default => {}
 attribute :instances, :kind_of => Array, :required => false, :default => []
+attribute :logs, :kind_of => Array, :required => false, :default => []
 attribute :version, :kind_of => Integer, :required => false, :default => nil
 attribute :use_integration_template, :kind_of => [TrueClass, FalseClass], :required => false, :default => false

spec/integrations/activemq_spec.rb

@@ -1,5 +1,6 @@
 describe 'datadog::activemq' do
   expected_yaml = <<-EOF
+    logs: ~
     instances:
     - host: localhost
       port: 1099