FIX: Logstash WARN logstash.outputs.elasticsearch on searchnode #10291
Comments
Sign up for free
to subscribe to this conversation on GitHub.
Already have an account?
Sign in.
{{ message }}
This is seen in the logstash.log on a searchnode.
[2023-05-05T14:00:07,425][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. status: 400, action: ["create", {:_id=>nil, :_index=>"logs-zeek-so", :routing=>nil, :pipeline=>"zeek.dns"}, {"message"=>"877,\"id.resp_h\":\"158.43.128.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":63257,\"rtt\":0.00011086463928222656,\"query\":\"update.microsoft.com\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":1,\"qtype_name\":\"A\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":true,\"RA\":true,\"Z\":0,\"answers\":[\"update.microsoft.com.nsatc.net\",\"65.55.25.61\"],\"TTLs\":[2069.0,44.0],\"rejected\":false}", "container"=>{"id"=>"dns.log"}, "log"=>{"file"=>{"path"=>"/nsm/zeek/logs/current/dns.log"}, "offset"=>10867}, "data_stream"=>{"namespace"=>"so", "type"=>"logs", "dataset"=>"zeek"}, "host"=>{"ip"=>["10.66.166.180", "fe80::98bb:17ff:fe3c:2b0", "172.17.1.1", "172.17.0.1"], "name"=>"jppsen2", "containerized"=>false, "architecture"=>"x86_64", "hostname"=>"jppsen2", "mac"=>["02-42-41-B9-BA-11", "02-42-A1-90-72-CF", "5A-DB-7C-97-C9-C6", "96-7F-24-0D-8A-E6", "9A-8D-31-20-3B-30", "9A-BB-17-3C-02-B0", "9E-D5-78-CF-95-09", "A2-88-FA-88-50-C7", "AA-C3-0E-D0-D9-86", "AE-C5-82-6A-76-FC", "FA-06-F2-38-55-CF"], "os"=>{"version"=>"9.1 (Blue Onyx)", "name"=>"Rocky Linux", "kernel"=>"5.14.0-162.23.1.el9_1.x86_64", "family"=>"redhat", "type"=>"linux", "codename"=>"Blue Onyx", "platform"=>"rocky"}, "id"=>"c1ab62dbe9af45e0a9a3f92f20ac083f"}, "type"=>"redis-input", "pipeline"=>"dns", "@timestamp"=>2023-05-05T14:00:05.998Z, "ecs"=>{"version"=>"8.0.0"}, "event"=>{"category"=>"network", "dataset"=>"zeek", "module"=>"zeek"}, "agent"=>{"version"=>"8.7.0", "type"=>"filebeat", "id"=>"70936732-1875-4b18-bd67-c78704b7a2e6", "ephemeral_id"=>"9c2ed582-a6a1-4351-b659-a99249e4174c", "name"=>"jppsen2"}, "input"=>{"type"=>"log"}, "metadata"=>{"beat"=>"filebeat", "version"=>"8.7.0", "input_id"=>"logfile-logs-d69973fb-22ed-4e03-abce-a19e7afe363e", "input"=>{"beats"=>{"host"=>{"ip"=>"10.66.166.180"}}}, "raw_index"=>"logs-zeek-so", "stream_id"=>"logfile-log.log-d69973fb-22ed-4e03-abce-a19e7afe363e", "type"=>"_doc", "pipeline"=>"zeek.dns"}, "@version"=>"1", "elastic_agent"=>{"version"=>"8.7.0", "id"=>"70936732-1875-4b18-bd67-c78704b7a2e6", "snapshot"=>false}, "tags"=>["elastic-agent", "beats_input_codec_plain_applied"]}], response: {"create"=>{"_index"=>"logs-zeek-so", "_id"=>nil, "status"=>400, "error"=>{"type"=>"script_exception", "reason"=>"runtime error", "script_stack"=>["ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", " ^---- HERE"], "script"=>"ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "lang"=>"painless", "position"=>{"offset"=>7, "start"=>0, "end"=>63}, "caused_by"=>{"type"=>"null_pointer_exception", "reason"=>"cannot access method/field [query] from a null def reference"}}}}[2023-05-05T14:44:38,354][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. status: 400, action: ["create", {:_id=>nil, :_index=>"logs-zeek-so", :routing=>nil, :pipeline=>"zeek.ntp"}, {"message"=>"{\"ts\":1683297869.420801,\"uid\":\"C3EmNt3KgodKTLxqp7\",\"id.orig_h\":\"192.168.1.95\",\"id.orig_p\":123,\"id.resp_h\":\"17.253.4.253\",\"id.resp_p\":123,\"version\":4,\"mode\":3,\"stratum\":0,\"poll\":256.0,\"precision\":1.0,\"root_delay\":0.0,\"root_disp\":0.0,\"ref_id\":\"\\\\x00\\\\x00\\\\x00\\\\x00\",\"ref_time\":0.0,\"org_time\":0.0,\"rec_time\":0.0,\"xmt_time\":1476535656.467835,\"num_exts\":0}", "container"=>{"id"=>"ntp.log"}, "log"=>{"file"=>{"path"=>"/nsm/zeek/logs/current/ntp.log"}, "offset"=>375704}, "data_stream"=>{"namespace"=>"so", "type"=>"logs", "dataset"=>"zeek"}, "host"=>{"ip"=>["10.66.166.180", "fe80::98bb:17ff:fe3c:2b0", "172.17.1.1", "172.17.0.1"], "name"=>"jppsen2", "containerized"=>false, "architecture"=>"x86_64", "id"=>"c1ab62dbe9af45e0a9a3f92f20ac083f", "os"=>{"version"=>"9.1 (Blue Onyx)", "name"=>"Rocky Linux", "kernel"=>"5.14.0-162.23.1.el9_1.x86_64", "family"=>"redhat", "type"=>"linux", "codename"=>"Blue Onyx", "platform"=>"rocky"}, "mac"=>["02-42-41-B9-BA-11", "02-42-A1-90-72-CF", "5A-DB-7C-97-C9-C6", "96-7F-24-0D-8A-E6", "9A-8D-31-20-3B-30", "9A-BB-17-3C-02-B0", "9E-D5-78-CF-95-09", "A2-88-FA-88-50-C7", "AA-C3-0E-D0-D9-86", "AE-C5-82-6A-76-FC", "FA-06-F2-38-55-CF"], "hostname"=>"jppsen2"}, "type"=>"redis-input", "pipeline"=>"ntp", "@timestamp"=>2023-05-05T14:44:36.428Z, "ecs"=>{"version"=>"8.0.0"}, "event"=>{"category"=>"network", "dataset"=>"zeek", "module"=>"zeek"}, "agent"=>{"version"=>"8.7.0", "type"=>"filebeat", "id"=>"70936732-1875-4b18-bd67-c78704b7a2e6", "ephemeral_id"=>"9c2ed582-a6a1-4351-b659-a99249e4174c", "name"=>"jppsen2"}, "input"=>{"type"=>"log"}, "metadata"=>{"beat"=>"filebeat", "version"=>"8.7.0", "input_id"=>"logfile-logs-d69973fb-22ed-4e03-abce-a19e7afe363e", "input"=>{"beats"=>{"host"=>{"ip"=>"10.66.166.180"}}}, "raw_index"=>"logs-zeek-so", "stream_id"=>"logfile-log.log-d69973fb-22ed-4e03-abce-a19e7afe363e", "type"=>"_doc", "pipeline"=>"zeek.ntp"}, "@version"=>"1", "elastic_agent"=>{"version"=>"8.7.0", "id"=>"70936732-1875-4b18-bd67-c78704b7a2e6", "snapshot"=>false}, "tags"=>["elastic-agent", "beats_input_codec_plain_applied"]}], response: {"create"=>{"_index"=>"logs-zeek-so", "_id"=>nil, "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"pipeline with id [zeek.ntp] does not exist"}}}[2023-05-05T14:37:13,262][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. status: 400, action: ["create", {:_id=>nil, :_index=>"logs-zeek-so", :routing=>nil, :pipeline=>"zeek.known_services"}, {"message"=>"{\"ts\":1683297120.8774967,\"host\":\"192.168.10.125\",\"port_num\":1274,\"port_proto\":\"tcp\",\"service\":[\"\"]}", "container"=>{"id"=>"known_services.log"}, "log"=>{"file"=>{"path"=>"/nsm/zeek/logs/current/known_services.log"}, "offset"=>693}, "data_stream"=>{"namespace"=>"so", "type"=>"logs", "dataset"=>"zeek"}, "host"=>{"ip"=>["10.66.166.197", "fe80::eca3:85ff:feac:7567", "172.17.1.1", "172.17.0.1"], "name"=>"jppsen3", "containerized"=>false, "architecture"=>"x86_64", "hostname"=>"jppsen3", "os"=>{"version"=>"9.1 (Blue Onyx)", "name"=>"Rocky Linux", "kernel"=>"5.14.0-162.23.1.el9_1.x86_64", "family"=>"redhat", "type"=>"linux", "codename"=>"Blue Onyx", "platform"=>"rocky"}, "mac"=>["02-42-D3-10-61-A1", "02-42-EB-58-1F-ED", "32-08-BF-A3-FE-2E", "4E-3B-7B-71-67-CA", "72-16-1B-79-37-09", "86-62-FF-91-34-51", "AA-F2-D0-3D-0B-95", "C2-AE-F6-17-90-3F", "D2-47-29-6B-33-1C", "E2-28-2C-53-30-90", "EE-A3-85-AC-75-67"], "id"=>"c1ab62dbe9af45e0a9a3f92f20ac083f"}, "type"=>"redis-input", "pipeline"=>"known_services", "@timestamp"=>2023-05-05T14:37:06.993Z, "ecs"=>{"version"=>"8.0.0"}, "event"=>{"category"=>"network", "dataset"=>"zeek", "module"=>"zeek"}, "agent"=>{"type"=>"filebeat", "version"=>"8.7.0", "id"=>"9bb435d1-cf95-4999-9691-5878f36ea20e", "ephemeral_id"=>"cafb9eb4-2539-459f-a12c-6fdbd9febd0d", "name"=>"jppsen3"}, "input"=>{"type"=>"log"}, "metadata"=>{"beat"=>"filebeat", "version"=>"8.7.0", "input_id"=>"logfile-logs-d69973fb-22ed-4e03-abce-a19e7afe363e", "input"=>{"beats"=>{"host"=>{"ip"=>"10.66.166.197"}}}, "raw_index"=>"logs-zeek-so", "stream_id"=>"logfile-log.log-d69973fb-22ed-4e03-abce-a19e7afe363e", "type"=>"_doc", "pipeline"=>"zeek.known_services"}, "@version"=>"1", "elastic_agent"=>{"version"=>"8.7.0", "id"=>"9bb435d1-cf95-4999-9691-5878f36ea20e", "snapshot"=>false}, "tags"=>["elastic-agent", "beats_input_codec_plain_applied"]}], response: {"create"=>{"_index"=>"logs-zeek-so", "_id"=>nil, "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"pipeline with id [zeek.known_services] does not exist"}}}[2023-05-05T13:54:44,316][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. status: 400, action: ["create", {:_id=>nil, :_index=>"logs-zeek-so", :routing=>nil, :pipeline=>"zeek.known_hosts"}, {"message"=>"{\"ts\":1683294880.248399,\"host\":\"192.168.3.65\"}", "container"=>{"id"=>"known_hosts.log"}, "log"=>{"file"=>{"path"=>"/nsm/zeek/logs/current/known_hosts.log"}, "offset"=>780}, "data_stream"=>{"namespace"=>"so", "type"=>"logs", "dataset"=>"zeek"}, "host"=>{"ip"=>["10.66.166.197", "fe80::eca3:85ff:feac:7567", "172.17.1.1", "172.17.0.1"], "name"=>"jppsen3", "containerized"=>false, "architecture"=>"x86_64", "hostname"=>"jppsen3", "os"=>{"version"=>"9.1 (Blue Onyx)", "name"=>"Rocky Linux", "kernel"=>"5.14.0-162.23.1.el9_1.x86_64", "family"=>"redhat", "type"=>"linux", "codename"=>"Blue Onyx", "platform"=>"rocky"}, "mac"=>["02-42-D3-10-61-A1", "02-42-EB-58-1F-ED", "32-08-BF-A3-FE-2E", "4E-3B-7B-71-67-CA", "72-16-1B-79-37-09", "86-62-FF-91-34-51", "AA-F2-D0-3D-0B-95", "C2-AE-F6-17-90-3F", "D2-47-29-6B-33-1C", "E2-28-2C-53-30-90", "EE-A3-85-AC-75-67"], "id"=>"c1ab62dbe9af45e0a9a3f92f20ac083f"}, "type"=>"redis-input", "pipeline"=>"known_hosts", "@timestamp"=>2023-05-05T13:54:42.656Z, "ecs"=>{"version"=>"8.0.0"}, "event"=>{"category"=>"network", "dataset"=>"zeek", "module"=>"zeek"}, "agent"=>{"type"=>"filebeat", "version"=>"8.7.0", "id"=>"9bb435d1-cf95-4999-9691-5878f36ea20e", "ephemeral_id"=>"cafb9eb4-2539-459f-a12c-6fdbd9febd0d", "name"=>"jppsen3"}, "input"=>{"type"=>"log"}, "metadata"=>{"beat"=>"filebeat", "version"=>"8.7.0", "input_id"=>"logfile-logs-d69973fb-22ed-4e03-abce-a19e7afe363e", "input"=>{"beats"=>{"host"=>{"ip"=>"10.66.166.197"}}}, "raw_index"=>"logs-zeek-so", "stream_id"=>"logfile-log.log-d69973fb-22ed-4e03-abce-a19e7afe363e", "type"=>"_doc", "pipeline"=>"zeek.known_hosts"}, "@version"=>"1", "elastic_agent"=>{"version"=>"8.7.0", "id"=>"9bb435d1-cf95-4999-9691-5878f36ea20e", "snapshot"=>false}, "tags"=>["elastic-agent", "beats_input_codec_plain_applied"]}], response: {"create"=>{"_index"=>"logs-zeek-so", "_id"=>nil, "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"pipeline with id [zeek.known_hosts] does not exist"}}}The text was updated successfully, but these errors were encountered: