td-agent.conf

####
## Output descriptions:
##

# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
<match td.*.*>
  type tdlog
  apikey YOUR_API_KEY

  auto_create_table
  buffer_type file
  buffer_path /var/log/td-agent/buffer/td
</match>

## match tag=debug.** and dump to console
<match debug.**>
  type stdout
</match>

####
## Source descriptions:
##

## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
<source>
  type forward
</source>

## built-in UNIX socket input
#<source>
#  type unix
#</source>

# HTTP input
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
<source>
  type http
  port 8118
</source>

## live debugging agent
<source>
  type debug_agent
  bind 127.0.0.1
  port 24230
</source>

####
## Theft Market Matches:
##

<match hdfs.tmp.*>
  type webhdfs
  host ip-172-31-15-74.us-west-1.compute.internal
  port 50070
  #path /log/%Y%m%d_%H/access.log.${hostname}
  path /tmp/fluentd  
  flush_interval 10s
</match>

<match hdfs.state.*>
  type webhdfs
  host ip-172-31-15-74.us-west-1.compute.internal
  port 50070
  path /data/state/json/%Y%m%d_%H/raw_state.log  
  flush_interval 60s
</match>

<match hdfs.city.*>
  type webhdfs
  host ip-172-31-15-74.us-west-1.compute.internal
  port 50070
  path /data/city/json/%Y%m%d_%H/raw_city.log  
  flush_interval 60s
</match>

<match hdfs.zipcode.*>
  type webhdfs
  host ip-172-31-15-74.us-west-1.compute.internal
  port 50070
  path /data/zipcode/json/%Y%m%d_%H/raw_zipcode.log  
  flush_interval 60s
</match>

<match hdfs.county.*>
  type webhdfs
  host ip-172-31-15-74.us-west-1.compute.internal
  port 50070
  path /data/county/json/%Y%m%d_%H/raw_county.log  
  flush_interval 60s
</match>