fluent.conf.rt

<source>
  @type tail
  @id access_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log.pos"
  tag jfrog.rt.access.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id artifactory_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log.pos"
  tag jfrog.rt.artifactory.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id frontend_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos"
  tag jfrog.rt.frontend.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log.pos"
  tag jfrog.rt.metadata.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos"
  tag jfrog.rt.router.service
  <parse>
    @type none
  </parse>
</source>
# Strip out color codes then field extract the service fields
<filter jfrog.rt.**.service>
  @type record_transformer
  enable_ruby true
  <record>
    message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
  </record>
</filter>
<filter jfrog.rt.**.service>
  @type parser
  key_name message
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
  </parse>
</filter>
# End Service Fields Extraction


<source>
  @type tail
  @id router_traefik_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos"
  tag jfrog.rt.router.traefik
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
  </parse>
</source>

<source>
  @type tail
  @id access_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log.pos"
  tag jfrog.rt.access.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
  </parse>
</source>
<source>
  @type tail
  @id artifactory_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log.pos"
  tag jfrog.rt.artifactory.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
  </parse>
</source>
<filter jfrog.rt.artifactory.request>
  @type record_transformer
  enable_ruby true
  <record>
    repo ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") ? (record["request_url"].split('/')[3]) : ("")}
    image ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2")  ? (record["request_url"].split('/')[5]) : ("")}
  </record>
</filter>
<source>
  @type tail
  @id frontend_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log.pos"
  tag jfrog.rt.frontend.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
  </parse>
</source>
<source>
  @type tail
  @id metadata_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log.pos"
  tag jfrog.rt.metadata.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
  </parse>
</source>
<source>
  @type tail
  @id router_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
  tag jfrog.rt.router.request
  <parse>
    @type json
  </parse>
</source>
<source>
  @type tail
  @id artifactory_access_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log.pos"
  tag jfrog.rt.artifactory.access
  <parse>
    @type regexp
    expression /^(?<timestamp>[^ ]*) \[(?<trace_id>[^\]]*)\] \[(?<action_response>[^\]]*)\] (?<repo_path>.*) for client : (?<username>.+)/(?<ip>.+).$/
  </parse>
</source>
<source>
  @type tail
  @id access_security_audit_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log.pos"
  tag jfrog.rt.access.audit
  <parse>
    @type regexp
    expression /^(?<timestamp>[^ ]*)\|(?<token_id>[^ ]*)\|(?<user_ip>[^ ]*)\|(?<user>[^ ]*)\|(?<logged_principal>[^ ]*)\|(?<entity_name>[^ ]*)\|(?<event_type>[^ ]*)\|(?<event>[^ ]*)\|(?<data_changed>.*)/
  </parse>
</source>
<filter jfrog.**>
  @type record_transformer
  <record>
    hostname "#{Socket.gethostname}"
    log_source ${tag}
  </record>
</filter>
<source>
  @type exec
  tag jfrog.callhome
  command "curl --request GET 'http://localhost:8081/artifactory/api/system/version' -H 'Authorization: Bearer <TOKEN>'"
  run_interval 1d
  <parse>
    @type json
  </parse>
</source>
<filter jfrog.callhome>
  @type record_transformer
  renew_record true
  keep_keys 'productId,features'
  enable_ruby true
  <record>
    productId 'jfrogLogAnalyticsElastic/0.9.0'
    features ${return([{"featureId" => "ArtifactoryVersion/" + record["version"]}])}
  </record>
</filter>
<match jfrog.callhome>
  @type http
  endpoint http://localhost:8081/artifactory/api/system/usage
  open_timeout 5
  content_type application/json
  headers {"Authorization":"Bearer <TOKEN>"}
  <format>
    @type json
  </format>
  <buffer>
    flush_interval 5s
  </buffer>
</match>
<filter jfrog.rt.artifactory.request>
  @type record_transformer
  enable_ruby true
  <record>
    repo ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") ? (record["request_url"].split('/')[3]) : ("")}
    image ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2")  ? (record["request_url"].split('/')[5]) : ("")}
  </record>
</filter>
<filter jfrog.rt.artifactory.request>
    @type record_transformer
    enable_ruby true
    <record>
      response_content_length_2 ${record["response_content_length"].to_f}
      request_content_length_2 ${record["request_content_length"].to_f}
    </record>
</filter>

###JFROG METRICS Plugin
<source>
  @type jfrog_metrics
  @id metrics_http_jfrt
  tag jfrog.metrics.artifactory
  interval 5s
  metric_prefix 'jfrog.artifactory'
  jpd_url "#{ENV['JF_JPD_URL']}"
  username "#{ENV['JF_JPD_USER_NAME']}"
  apikey "#{ENV['JF_JPD_API_KEY']}"
  token "#{ENV['JF_JPD_TOKEN']}"
  target_platform "ELASTIC"
</source>


########################
# ELASTIC OUTPUT METRICS
########################
<match jfrog.metrics.**>
  @type elasticsearch
  @id elasticsearch_metrics
  host "#{ENV['ELASTIC_HOST']}"
  port "#{ENV['ELASTIC_PORT']}"
  user "#{ENV['ELASTIC_USER']}"
  password "#{ENV['ELASTIC_PASSWORD']}"
  scheme "#{ENV['ELASTIC_SCHEME']}"
  ssl_verify "#{ENV['ELASTIC_SSL_VERIFY']}"
  index_name unified-metrics-artifactory
  include_tag_key true
  type_name fluentd
  logstash_format false
  include_timestamp true
</match>
#END ELASTIC OUTPUT

########################
# ELASTIC OUTPUT LOGS
########################
<match jfrog.**>
  @type elasticsearch
  @id elasticsearch_logs
  host "#{ENV['ELASTIC_HOST']}"
  port "#{ENV['ELASTIC_PORT']}"
  user "#{ENV['ELASTIC_USER']}"
  password "#{ENV['ELASTIC_PASSWORD']}"
  scheme "#{ENV['ELASTIC_SCHEME']}"
  ssl_verify "#{ENV['ELASTIC_SSL_VERIFY']}"
  index_name unified-artifactory
  include_tag_key true
  type_name fluentd
  logstash_format false
</match>
#END ELASTIC OUTPUT