####
## Output descriptions:
##

# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
<match td.*.*>
  type tdlog
  apikey YOUR_API_KEY

  auto_create_table
  buffer_type file
  buffer_path /var/log/td-agent/buffer/td

  <secondary>
    type file
    path /var/log/td-agent/failed_records
  </secondary>
</match>

## match tag=debug.** and dump to console
<match debug.**>
  type stdout
</match>

####
## Source descriptions:
##

## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
<source>
  type forward
</source>

## built-in UNIX socket input
#<source>
#  type unix
#</source>

# HTTP input
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
<source>
  type http
  port 8888
</source>

## live debugging agent
<source>
  type debug_agent
  bind 127.0.0.1
  port 24230
</source>

####
## Examples:
##

## File input
## read apache logs continuously and tags td.apache.access
#<source>
#  type tail
#  format apache
#  path /var/log/httpd-access.log
#  tag td.apache.access
#</source>

<source>
  type tail
  format apache
  time_format %d/%b/%Y:%H:%M:%S %z
  pos_file /var/log/td-agent/access_log.pos
  path /var/log/httpd/access_log
  tag piwiktracker.apache.access
</source>

<filter piwiktracker.apache.access>
    type grep
    regexp1 path /piwik/piwik\.php\?action_name=.*\&idsite=\d+
</filter>

<match piwiktracker.apache.access>
    type record_reformer
    tag piwiktracker.apache.access.decode
    action_name ${path[/piwik\.php\?action_name=([^\&]+)/,1]}
    idsite ${path[/piwik\.php\?action_name=.*\&idsite=(\d+)/,1]}
    rec ${path[/piwik\.php\?action_name=.*\&rec=(\d+)/,1]}
    r ${path[/piwik\.php\?action_name=.*\&r=(\d+)/,1]}
    h ${path[/piwik\.php\?action_name=.*\&h=(\d+)/,1]}
    m ${path[/piwik\.php\?action_name=.*\&m=(\d+)/,1]}
    s ${path[/piwik\.php\?action_name=.*\&s=(\d+)/,1]}
    url ${path[/piwik\.php\?action_name=.*\&url=([^\&]+)/,1]}
    urlref ${path[/piwik\.php\?action_name=.*\&urlref=([^\&]+)/,1]}
    piwikid ${path[/piwik\.php\?action_name=.*\&_id=([a-z\d]+)/,1]}
    idts ${path[/piwik\.php\?action_name=.*\&_idts=([\d]+)/,1] == "1" ? "true" : "false" }
    idvc ${path[/piwik\.php\?action_name=.*\&_idvc=(\d)/,1] == "1" ? "true" : "false" }
    idn ${path[/piwik\.php\?action_name=.*\&_idn=(\d)/,1] == "1" ? "true" : "false" }
    refts ${path[/piwik\.php\?action_name=.*\&_refts=(\d+)/,1]}
    viewts ${path[/piwik\.php\?action_name=.*\&_viewts=(\d+)/,1]}
    ref ${path[/piwik\.php\?action_name=.*\&_ref=([^\&]+)/,1]}
    send_image ${path[/piwik\.php\?action_name=.*\&send_image=(\d+)/,1]}
    pdf ${path[/piwik\.php\?action_name=.*\&pdf=(\d+)/,1] == "1" ? "true" : "false" }
    qt ${path[/piwik\.php\?action_name=.*\&qt=(\d+)/,1] == "1" ? "true" : "false" }
    realp ${path[/piwik\.php\?action_name=.*\&realp=(\d+)/,1] == "1" ? "true" : "false" }
    wma ${path[/piwik\.php\?action_name=.*\&wma=(\d+)/,1] == "1" ? "true" : "false" }
    dir ${path[/piwik\.php\?action_name=.*\&dir=(\d+)/,1] == "1" ? "true" : "false" }
    fla ${path[/piwik\.php\?action_name=.*\&fla=(\d+)/,1] == "1" ? "true" : "false" }
    java ${path[/piwik\.php\?action_name=.*\&java=(\d+)/,1] == "1" ? "true" : "false" }
    gears ${path[/piwik\.php\?action_name=.*\&gears=(\d+)/,1] == "1" ? "true" : "false" }
    ag ${path[/piwik\.php\?action_name=.*\&ag=(\d+)/,1] == "1" ? "true" : "false" }
    cookie ${path[/piwik\.php\?action_name=.*\&cookie=(\d+)/,1] == "1" ? "true" : "false" }
    res ${path[/piwik\.php\?action_name=.*\&res=([\d\.]+x[\d+\.]+)/,1]}
    gt_ms ${path[/piwik\.php\?action_name=.*\&gt_ms=(\d+)/,1]}
</match>

<filter piwiktracker.apache.access.decode>
  type geoip
  geoip_lookup_key host
  skip_adding_null_record  true
  geoip_database "/usr/share/GeoIP/GeoLiteCity.dat"
  flush_interval 1s
  <record>
    country ${country_code["host"]}
    location ${latitude['host']},${longitude["host"]}
    country_code3 ${country_code3["host"]}
    country ${country_code["host"]}
    country_name ${country_name["host"]}
    city ${city["host"]}
  </record>
</filter>

<match piwiktracker.apache.access.decode>
    type  uri_decode
    tag   piwiktracker.apache.access.store
    #tag debug.${tag}
    key_names action_name,ref,url,urlref
</match>

## File output
## match tag=local.** and write to file
#<match local.**>
#  type file
#  path /var/log/td-agent/access
#</match>

## Forwarding
## match tag=system.** and forward to another td-agent server
#<match system.**>
#  type forward
#  host 192.168.0.11
#  # secondary host is optional
#  <secondary>
#    host 192.168.0.12
#  </secondary>
#</match>

## Multiple output
## match tag=td.*.* and output to Treasure Data AND file
#<match td.*.*>
#  type copy
#  <store>
#    type tdlog
#    apikey API_KEY
#    auto_create_table
#    buffer_type file
#    buffer_path /var/log/td-agent/buffer/td
#  </store>
#  <store>
#    type file
#    path /var/log/td-agent/td-%Y-%m-%d/%H.log
#  </store>
#</match>

<match piwiktracker.apache.access.store>
    type copy
    <store>
        type elasticsearch
        type_name access_log
        host 127.0.0.1
        port 9200
        logstash_format true
        logstash_prefix apache-log
        logstash_dateformat %Y%m%d
        include_tag_key true
        tag_key @log_name
        flush_interval 10s
    </store>
#    <store>
#        type file
#        path /tmp/fluentd-debug.log
#    </store>
</match>