project_template.go

package generator

var ProjectTemplate = `{{range $i, $store := .projectTargets -}}
{{ if $store.CurrentTarget }}
<source>
   @type  tail
   path  /var/log/containers/*.log
   pos_file  /fluentd/log/fluentd-project-{{$store.ProjectName}}-logging.pos
   time_format  %Y-%m-%dT%H:%M:%S
   tag  {{$store.ProjectName}}.*
   format  json
   read_from_head  true
</source>

<filter  {{$store.ProjectName}}.**>
   @type  kubernetes_metadata
   merge_json_log  true
   preserve_json_log  true
</filter>

<filter {{$store.ProjectName}}.**>
  @type record_transformer
  enable_ruby  true
  <record>
    tag ${tag}
    namespace ${record["kubernetes"]["namespace_name"]}
    projectID {{$store.ProjectName}}
  </record>
</filter>

{{- template "filter-custom-tags" $store -}}

<filter {{$store.ProjectName}}.**>
  @type grep
  <regexp>
    key namespace
    pattern {{$store.GrepNamespace}}
  </regexp>
</filter>

<filter {{$store.ProjectName}}.**>
  @type record_transformer
  remove_keys namespace
</filter>

{{template "project-filter-syslog" $store}}
{{template "project-match" $store}}
{{end -}}
{{end -}}
`

var projectFilterSyslogTemplate = `
{{define "project-filter-syslog"}}
{{$store := .}}
{{ if eq $store.CurrentTarget "syslog"}}
{{ if $store.SyslogConfig.Token}}
<filter {{$store.ProjectName}}.** project-custom.{{$store.ProjectName}}.**>
  @type record_transformer
  <record>
    tag ${tag} {{$store.SyslogConfig.Token}}
  </record>
</filter>
{{end -}}
{{end -}}
{{end -}}
`
var projectOutputTemplate = ` 
{{define "project-match"}}
{{$store := .}}
<match  {{$store.ProjectName}}.** project-custom.{{$store.ProjectName}}.**> 
    {{ if eq $store.CurrentTarget "elasticsearch"}}
    @type elasticsearch
    include_tag_key  true
    {{- if and $store.ElasticsearchConfig.AuthUserName $store.ElasticsearchConfig.AuthPassword}}
    user {{$store.ElasticsearchConfig.AuthUserName}}
    password {{$store.ElasticsearchConfig.AuthPassword}}
    {{- end }}
    hosts {{$store.ElasticsearchConfig.Endpoint}}    

    logstash_prefix "{{$store.ElasticsearchConfig.IndexPrefix}}"
    logstash_format true
    logstash_dateformat  {{$store.WrapElasticsearch.DateFormat}}
    type_name  "container_log"

    {{ if eq $store.WrapElasticsearch.Scheme "https"}}
    ssl_verify {{$store.ElasticsearchConfig.SSLVerify}}
    ssl_version {{ $store.ElasticsearchConfig.SSLVersion }}

    {{ if $store.ElasticsearchConfig.Certificate }}
    ca_file /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_ca.pem
    {{end -}}

    {{ if and $store.ElasticsearchConfig.ClientCert $store.ElasticsearchConfig.ClientKey}}
    client_cert /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-cert.pem
    client_key /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-key.pem
    {{end -}}

    {{ if $store.ElasticsearchConfig.ClientKeyPass}}
    client_key_pass {{$store.ElasticsearchConfig.ClientKeyPass}}
    {{end -}}
    {{end -}}
    {{end -}}

    {{ if eq $store.CurrentTarget "splunk"}}
    @type splunk_hec
    host {{$store.WrapSplunk.Host}}
    port {{$store.WrapSplunk.Port}}
    token {{$store.SplunkConfig.Token}}
    {{ if $store.SplunkConfig.Source}}
    sourcetype {{$store.SplunkConfig.Source}}
    {{end -}}
    {{ if $store.SplunkConfig.Index}}
    default_index {{ $store.SplunkConfig.Index }}
    {{end -}}

    {{ if eq $store.WrapSplunk.Scheme "https"}}
    use_ssl true    
    ssl_verify {{$store.SplunkConfig.SSLVerify}}    

    {{ if $store.SplunkConfig.Certificate }}    
    ca_file /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_ca.pem
    {{end -}}

    {{ if and $store.SplunkConfig.ClientCert $store.SplunkConfig.ClientKey}}    
    client_cert /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-cert.pem
    client_key /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-key.pem
    {{end -}}

    {{ if $store.SplunkConfig.ClientKeyPass}}    
    client_key_pass {{ $store.SplunkConfig.ClientKeyPass }}
    {{end -}}
    {{end -}}
    {{end -}}

    {{ if eq $store.CurrentTarget "kafka"}}
    @type kafka_buffered
    {{ if $store.KafkaConfig.ZookeeperEndpoint }}
    zookeeper {{$store.WrapKafka.Zookeeper}}
    {{else}}
    brokers {{$store.WrapKafka.Brokers}}
    {{end}}
    default_topic {{$store.KafkaConfig.Topic}}
    output_data_type  "json"
    output_include_tag  true
    output_include_time  true
    # get_kafka_client_log  true
    max_send_retries 3
    
    {{ if $store.KafkaConfig.Certificate }}        
    ssl_ca_cert /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_ca.pem
    {{end}}

    {{ if and $store.KafkaConfig.ClientCert $store.KafkaConfig.ClientKey}}        
    ssl_client_cert /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-cert.pem
    ssl_client_cert_key /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-key.pem
    {{ end -}}
    {{end -}}

    {{ if eq $store.CurrentTarget "syslog"}}
    @type remote_syslog
    host {{$store.WrapSyslog.Host}}
    port {{$store.WrapSyslog.Port}}
    severity {{$store.SyslogConfig.Severity}}
    protocol {{$store.SyslogConfig.Protocol}}
    {{ if $store.SyslogConfig.Program }}
    program {{$store.SyslogConfig.Program}}
    {{end -}}
    packet_size 65535

    {{ if eq $store.SyslogConfig.SSLVerify true}}
    verify_mode 1
    {{else -}}
    verify_mode 0
    {{end -}}

    {{ if $store.SyslogConfig.Certificate }}
    tls true        
    ca_file /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_ca.pem
    {{end}}

    {{ if and $store.SyslogConfig.ClientCert $store.SyslogConfig.ClientKey}}        
    client_cert /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-cert.pem
    client_cert_key /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_client-key.pem
    {{ end -}}
    {{end}}

    {{ if eq $store.CurrentTarget "fluentforwarder"}}
    @type forward
    {{ if $store.FluentForwarderConfig.EnableTLS }}
    transport tls    
    tls_allow_self_signed_cert true
    tls_verify_hostname true
    {{end -}}
    {{ if $store.FluentForwarderConfig.Certificate }}
    tls_cert_path /fluentd/etc/ssl/project_{{$store.WrapProjectName}}_ca.pem
    {{end -}}  
    
    {{ if $store.FluentForwarderConfig.Compress }}
    compress gzip
    {{end -}}

    {{ if $store.WrapFluentForwarder.EnableShareKey }}
    <security>
      self_hostname "#{Socket.gethostname}"
      shared_key true
    </security>
    {{end -}}

    {{range $k, $val := $store.WrapFluentForwarder.FluentServers -}}
    <server>
      {{if $val.Hostname}}
      name {{$val.Hostname}}
      {{end -}}
      host {{$val.Host}}
      port {{$val.Port}}
      {{ if $val.SharedKey}}
      shared_key {{$val.SharedKey}}
      {{end -}}
      {{ if $val.Username}}
      username  {{$val.Username}}
      {{end -}}
      {{ if $val.Password}}
      password  {{$val.Password}}
      {{end -}}
      weight  {{$val.Weight}}
      {{if $val.Standby}}
      standby
      {{end -}}
    </server>
    {{end -}}
    {{end -}}   
    
    <buffer>
      @type file
      path /fluentd/etc/buffer/project.{{$store.WrapProjectName}}.buffer
      flush_interval {{$store.OutputFlushInterval}}s
      {{ if eq $store.CurrentTarget "splunk"}}
      buffer_chunk_limit 8m
      {{end -}}
    </buffer> 

    disable_retry_limit
    num_threads 8
    slow_flush_log_threshold 40.0
</match>
{{end -}}
`