fluent.conf.rt

# JFROG ARTIFACTORY METRICS SOURCE
<source>
  @type jfrog_metrics
  @id metrics_http_jfrt
  tag jfrog.metrics.artifactory
  metric_prefix 'jfrog.artifactory'
  jpd_url "#{ENV['JPD_URL']}"
  username "#{ENV['JPD_ADMIN_USERNAME']}"
  token "#{ENV['JFROG_ADMIN_TOKEN']}"
  common_jpd "#{ENV['COMMON_JPD']}"
  target_platform "DATADOG"
  execution_interval 60s
  request_timeout 30s
  # @log_level debug
  # verify_ssl "#{ENV['DATADOG_VERIFY_SSL']}"
</source>
<match jfrog.metrics.**>
  @type jfrog_send_metrics
  target_platform "DATADOG"
  apikey "#{ENV['DATADOG_API_KEY']}"
  url "https://api.#{ENV['DATADOG_API_HOST']}/api/v2/series"
  gzip_compression "#{ENV['DATADOG_COMPRESS_DATA']}"
  request_timeout 30s
  # @log_level debug
  # verify_ssl "#{ENV['DATADOG_VERIFY_SSL']}"
  # ddtags ["instance:test-artifactory", "cluster:GKE"]
</match>
# CALLHOME
<source>
  @type exec
  tag callhome
  command "curl --request GET '#{ENV['JPD_URL']}/artifactory/api/system/version' -H 'Authorization: Bearer #{ENV['JFROG_ADMIN_TOKEN']}'"
  run_interval 24h
  <parse>
    @type json
  </parse>
</source>
<match callhome>
  @type copy
  <store>
    @type relabel
    @label @jfrogcallhome
  </store>
  <store>
    @type relabel
    @label @heapcallhome
  </store>
</match>
<label @heapcallhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      app_id '251985503'
      identity "#{ENV['JPD_URL']}"
      event 'datadog-artifactory-fluentd-config'
      properties ${return({"featureId" => "ArtifactoryVersion/" + record["version"], "productId" => "OnPremObservability-Datadog/1.0.0"})}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "https://heapanalytics.com/api/track"
    open_timeout 5
    content_type application/json
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>
<label @jfrogcallhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      productId 'OnPremObservability-Datadog/1.0.0'
      features ${return([{"featureId" => "ArtifactoryVersion/" + record["version"]}])}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "#{ENV['JPD_URL']}/artifactory/api/system/usage"
    open_timeout 5
    content_type application/json
    <auth>
      method basic
      username "#{ENV['JPD_ADMIN_USERNAME']}"
      password "#{ENV['JFROG_ADMIN_TOKEN']}"
    </auth>
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>

# LOG SOURCE DIRECTIVES
## SERVICE LOGS
<source>
  @type tail
  @id access_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log.pos"
  tag jfrog.rt.access.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id artifactory_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log.pos"
  tag jfrog.rt.artifactory.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log.pos"
  tag jfrog.rt.metadata.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos"
  tag jfrog.rt.router.service
  <parse>
    @type none
  </parse>
</source>
## TRAEFIK LOGS
<source>
  @type tail
  @id router_traefik_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos"
  tag jfrog.rt.router.traefik
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
  </parse>
</source>
## REQUEST LOGS
<source>
  @type tail
  @id access_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log.pos"
  tag jfrog.rt.access.request
  <parse>
    @type regexp
    expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    types response_content_length:integer, request_content_length:integer, return_status:integer
  </parse>
</source>
<source>
  @type tail
  @id artifactory_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log.pos"
  tag jfrog.rt.artifactory.request
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id frontend_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log.pos"
  tag jfrog.rt.frontend.request
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log.pos"
  tag jfrog.rt.metadata.request
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
  tag jfrog.rt.router.request
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id artifactory_request_out_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log.pos"
  tag jfrog.rt.request.out
  read_from_head true
  <parse>
    @type regexp
    expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_repo_name>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<remote_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)$
  </parse>
</source>
## ACCESS LOG
<source>
  @type tail
  @id artifactory_access_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log.pos"
  tag jfrog.rt.artifactory.access
  <parse>
    @type regexp
    expression /^(?<log_timestamp>[^\s]*) \[(?<trace_id>[^\s\]]*)\s*\] \[(?<action_response>[^\]\r\n]*)\] (?<repo_path>[^\s]*)? *for client : *(?<username>[^\s]*) *\/ *(?<ip>[^\s]*)? ?(\[(?<token>[^\s]*)\])?$/
  </parse>
</source>
## AUDIT LOG
<source>
  @type tail
  @id access_security_audit_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log.pos"
  tag jfrog.rt.access.audit
  <parse>
    @type regexp
    expression /^(?<log_timestamp>[^ ]*)\|(?<token_id>[^ ]*)\|(?<user_ip>[^ ]*)\|(?<user>[^ ]*)\|(?<logged_principal>[^ ]*)\|(?<entity_name>[^ ]*)\|(?<event_type>[^ ]*)\|(?<event>[^ ]*)\|(?<data_changed>.*)/
  </parse>
</source>


# FILTER DIRECTIVE
## ALL LOGS
<filter **>
  @type concat
  key message
  multiline_start_regexp /\d{4}-\d{1,2}-\d{1,2}/
  timeout_label @NORMAL
  flush_interval 5
</filter>
<match **>
  @type relabel
  @label @NORMAL
</match>
<label @NORMAL>
## SERVICE LOGS
  <filter jfrog.rt.**.service>
    @type record_transformer
    enable_ruby true
    <record>
      message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
    </record>
  </filter>
  <filter jfrog.rt.access.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
    </parse>
  </filter>
  <filter jfrog.rt.artifactory.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
    </parse>
  </filter>
  <filter jfrog.rt.metadata.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
    </parse>
  </filter>
  <filter jfrog.rt.router.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
    </parse>
  </filter>
  ## REQUEST LOGS
  <filter jfrog.rt.artifactory.request>
    @type parser
    key_name message
    <parse>
      @type regexp
      expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    </parse>
  </filter>
  <filter jfrog.rt.artifactory.request>
    @type record_transformer
    enable_ruby true
    <record>
      repo ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") ? (record["request_url"].split('/')[3]) : ("")}
      image ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2")  ? (record["request_url"].split('/')[5]) : ("")}
    </record>
  </filter>
  <filter jfrog.rt.artifactory.request>
    @type record_transformer
    enable_ruby true
    <record>
      response_content_length ${record["response_content_length"].to_f}
      request_content_length ${record["request_content_length"].to_f}
    </record>
  </filter>
  <filter jfrog.rt.frontend.request>
    @type parser
    key_name message
    <parse>
      @type regexp
      expression ^(?<log_timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    </parse>
  </filter>
  <filter jfrog.rt.metadata.request>
    @type parser
    key_name message
    <parse>
      @type regexp
      expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    </parse>
  </filter>
  ## ACCESS LOGS
  <filter jfrog.rt.artifactory.access>
    @type record_transformer
    enable_ruby true
    <record>
      impacted_artifacts ${if record['repo_path'].length > 1; "default/" + record["repo_path"].split(':')[0] + "/" + record["repo_path"].split(':')[1].rstrip ; end;}
    </record>
  </filter>
  ## ALL LOGS
  <filter jfrog.**>
    @type record_transformer
    <record>
      hostname "#{Socket.gethostname}"
      log_source ${tag}
    </record>
  </filter>
  # TEST OUTPUT
  #<match **>
  #  @type stdout
  #</match>

  # DATADOG OUTPUT
  ## DATADOG ARTIFACTORY LOGS OUTPUT
  <match jfrog.**>
    @type datadog
    @id jfrog_artifactory
    api_key "#{ENV['DATADOG_API_KEY']}"
    #optional
    include_tag_key true
    dd_source jfrog_platform
    service jfrog_artifactory
    host "http-intake.logs.#{ENV['DATADOG_API_HOST']}"
    <buffer>
      flush_interval 1s
      # frequency of the buffer flush
      flush_thread_count 5
      # The number of threads to flush/write chunks in parallel
      chunk_limit_records 10
      # The max number of events that each chunks can store in it
    </buffer>
  </match>
</label>