Elasticsearch geo_point causes ERROR during insert into Elassandra with CURL

[albroscani]

Hi,
I'm trying to run the Kibana Getting Started tutorial with Elassandra 2.1.1-8 and Kibana 4.3.3 (linux-x64).
To accomplish the Map Visualization I have launched the following command:

****** ELASTICSEARCH COORDINATES DEFINITION ******

curl -XPUT http://localhost:9200/logstash_20150520 -d ' { "mappings": { "log": { "properties": { "geo": { "properties": { "coordinates": { "type": "geo_point" } } } } } } } ';

and then I've done a bulk insert with the following command:

****** ELASTICSEARCH BULK INSERT ******

curl -XPOST 'localhost:9200/_bulk?pretty' --data-binary @logs.jsonl

The bulk fails with the following ERROR (from system.log):

######## START ERROR

2016-04-28 13:08:41,244 ERROR [elasticsearch[localhost][index][T#4]] InternalCassandraClusterService.java:1479 insertDocument [localhost] [logstash_20150520].[log] failed to parse field geo={dest=IN, src=R
U, coordinates={lon=-87.44675972, lat=31.01621528}, srcdest=RU:IN}
java.lang.ClassCastException: null
2016-04-28 13:08:41,244 DEBUG [elasticsearch[localhost][index][T#4]] TransportReplicationAction.java:604 performOnPrimary [localhost] [logstash_20150520][0], node[c7610fa4-e08b-405b-b855-c682a935543a], [P]
, v[1], s[STARTED], a[id=B0BPlnTrS2-fmliQ7rwABw]: Failed to execute [index {[logstash_20150520][log][test], source[
{
"@timestamp" : "2015-05-01T15:57:34.915Z",
"ip" : "166.114.155.140",
"extension" : "jpg",
"response" : "200",
"geo" : {
"coordinates" : {
"lat" : 31.01621528,
"lon" : -87.44675972
},
"src" : "RU",
"dest" : "IN",
"srcdest" : "RU:IN"
},
"@tags" : ["success", "info"],
"utc_time" : "2015-05-20T15:57:34.915Z",
"referer" : "http://twitter.com/success/ellison-onizuka",
"agent" : "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24",
"clientip" : "166.114.155.140",
"bytes" : 6839,
"host" : "media-for-the-masses.theacademyofperformingartsandscience.org",
"request" : "/uploads/laurel-b-clark.jpg",
"url" : "https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/laurel-b-clark.jpg",
"@message" : "166.114.155.140 - - [2015-05-20T15:57:34.915Z] "GET /uploads/laurel-b-clark.jpg HTTP/1.1" 200 6839 "-" "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"",
"spaces" : "this is a thing with lots of spaces wwwwoooooo",
"xss" : "<script>console.log("xss")</script>",
"headings" : ["

daniel-burbank

", "http://twitter.com/success/john-grunsfeld"],
"links" : ["terry-virts@www.slate.com", "http://www.slate.com/info/curtis-brown", "www.www.slate.com"],
"relatedContent" : [],
"machine" : {
"os" : "win xp",
"ram" : 32212254720
},
"@Version" : "1"
}]}]
java.lang.ClassCastException: null
2016-04-28 13:08:41,244 TRACE [elasticsearch[localhost][index][T#4]] TransportReplicationAction.java:542 finishAsFailed [localhost] operation failed
java.lang.ClassCastException: null
2016-04-28 13:08:41,244 TRACE [elasticsearch[localhost][index][T#4]] ChildMemoryCircuitBreaker.java:182 addWithoutBreaking [localhost] [request] Adjusted breaker by [16440] bytes, now [16440]
2016-04-28 13:08:41,245 INFO [elasticsearch[localhost][index][T#4]] BytesRestResponse.java:131 convert /logstash_20150520/log/test Params: {id=test, index=logstash_20150520, type=log}
java.lang.ClassCastException: null

######## END ERROR

These are the index definition of Elasticsearch and the Keyspace definition of Cassandra

****** ELASTICSEARCH INDEX ******

"logstash_20150520" : {
"state" : "open",
"settings" : {
"index" : {
"creation_date" : "1461776568462",
"uuid" : "yx1IPFnmT5icKYTcMfZh7w",
"number_of_replicas" : "0",
"number_of_shards" : "1",
"version" : {
"created" : "2010199"
}
}
},
"mappings" : {
"log" : {
"properties" : {
"spaces" : {
"type" : "string"
},
"relatedContent" : {
"properties" : {
"articleTag" : {
"type" : "string"
},
"twitterCard" : {
"type" : "string"
},
"ogImageHeight" : {
"type" : "string"
},
"articlePublished_time" : {
"format" : "strict_date_optional_time||epoch_millis",
"type" : "date"
},
"twitterSite" : {
"type" : "string"
},
"ogDescription" : {
"type" : "string"
},
"url" : {
"type" : "string"
},
"articleModified_time" : {
"format" : "strict_date_optional_time||epoch_millis",
"type" : "date"
},
"ogType" : {
"type" : "string"
},
"twitterImage" : {
"type" : "string"
},
"ogImageWidth" : {
"type" : "string"
},
"ogUrl" : {
"type" : "string"
},
"ogTitle" : {
"type" : "string"
},
"ogImage" : {
"type" : "string"
},
"twitterTitle" : {
"type" : "string"
},
"ogSite_name" : {
"type" : "string"
},
"twitterDescription" : {
"type" : "string"
},
"articleSection" : {
"type" : "string"
}
}
},
"@message" : {
"type" : "string"
},
"bytes" : {
"type" : "long"
},
"geo" : {
"properties" : {
"dest" : {
"type" : "string"
},
"src" : {
"type" : "string"
},
"coordinates" : {
"type" : "geo_point"
},
"srcdest" : {
"type" : "string"
}
}
},
"host" : {
"type" : "string"
},
"clientip" : {
"type" : "string"
},
"@tags" : {
"type" : "string"
},
"xss" : {
"type" : "string"
},
"utc_time" : {
"format" : "strict_date_optional_time||epoch_millis",
"type" : "date"
},
"links" : {
"type" : "string"
},
"machine" : {
"properties" : {
"os" : {
"type" : "string"
},
"ram" : {
"type" : "long"
}
}
},
"@Version" : {
"type" : "string"
},
"agent" : {
"type" : "string"
},
"url" : {
"type" : "string"
},
"memory" : {
"type" : "long"
},
"phpmemory" : {
"type" : "long"
},
"ip" : {
"type" : "string"
},
"response" : {
"type" : "string"
},
"extension" : {
"type" : "string"
},
"headings" : {
"type" : "string"
},
"@timestamp" : {
"format" : "strict_date_optional_time||epoch_millis",
"type" : "date"
},
"request" : {
"type" : "string"
},
"referer" : {
"type" : "string"
}
}
}
},
"aliases" : [ ]
},

****** CASSANDRA KEYSPACE ******

CREATE KEYSPACE logstash_20150520 WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1': '1'} AND durable_writes = true;

CREATE TYPE logstash_20150520.log_geo_coordinates (
lat double,
lon double
);

CREATE TYPE logstash_20150520.log_geo (
coordinates frozen<list<frozen<log_geo_coordinates>>>,
srcdest frozen<list>,
dest frozen<list>,
src frozen<list>
);

CREATE TYPE logstash_20150520.log_machine (
ram frozen<list>,
os frozen<list>
);

CREATE TYPE logstash_20150520."log_relatedContent" (
"ogDescription" frozen<list>,
"twitterDescription" frozen<list>,
"ogSite_name" frozen<list>,
"articleSection" frozen<list>,
"articlePublished_time" frozen<list>,
"twitterSite" frozen<list>,
"ogTitle" frozen<list>,
"twitterTitle" frozen<list>,
"twitterImage" frozen<list>,
"ogImageHeight" frozen<list>,
"articleModified_time" frozen<list>,
url frozen<list>,
"ogType" frozen<list>,
"ogUrl" frozen<list>,
"ogImageWidth" frozen<list>,
"ogImage" frozen<list>,
"twitterCard" frozen<list>,
"articleTag" frozen<list>
);

CREATE TABLE logstash_20150520.log (
"_id" text PRIMARY KEY,
"@message" list,
"@tags" list,
"@timestamp" list,
"@Version" list,
agent list,
bytes list,
clientip list,
extension list,
geo list<frozen<log_geo>>,
headings list,
host list,
ip list,
links list,
machine list<frozen<log_machine>>,
memory list,
phpmemory list,
referer list,
"relatedContent" list<frozen<log_relatedContent>>,
request list,
response list,
spaces list,
url list,
utc_time list,
xss list
) WITH bloom_filter_fp_chance = 0.01
AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}'
AND comment = 'Auto-created by Elassandra'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'}
AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99.0PERCENTILE';
CREATE CUSTOM INDEX elastic_log_xss_idx ON logstash_20150520.log (xss) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_host_idx ON logstash_20150520.log (host) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log__message_idx ON logstash_20150520.log ("@message") USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_extension_idx ON logstash_20150520.log (extension) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_clientip_idx ON logstash_20150520.log (clientip) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_spaces_idx ON logstash_20150520.log (spaces) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_bytes_idx ON logstash_20150520.log (bytes) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_url_idx ON logstash_20150520.log (url) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log__version_idx ON logstash_20150520.log ("@Version") USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_geo_idx ON logstash_20150520.log (geo) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_ip_idx ON logstash_20150520.log (ip) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_relatedContent_idx ON logstash_20150520.log ("relatedContent") USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_memory_idx ON logstash_20150520.log (memory) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log__tags_idx ON logstash_20150520.log ("@tags") USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_request_idx ON logstash_20150520.log (request) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_machine_idx ON logstash_20150520.log (machine) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_response_idx ON logstash_20150520.log (response) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_headings_idx ON logstash_20150520.log (headings) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_referer_idx ON logstash_20150520.log (referer) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_links_idx ON logstash_20150520.log (links) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log__timestamp_idx ON logstash_20150520.log ("@timestamp") USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_phpmemory_idx ON logstash_20150520.log (phpmemory) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_utc_time_idx ON logstash_20150520.log (utc_time) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';
CREATE CUSTOM INDEX elastic_log_agent_idx ON logstash_20150520.log (agent) USING 'org.elasticsearch.cassandra.ElasticSecondaryIndex';

Maybe I'm doing something wrong?

If I don't run the command for the geo_point creation on the Elasticsearch index, the bulk insert goes well but lat and lon are simply doubles (not geo coordinates) and kibana can't show them over the Map.

Thanks.

Alberto

[vroyer]

Hi Alberto,
I have reproduce your issue, it's a bug with geo_point.
It's fixed and i'll deliver a new release this week.
Thanks,
Vincent.

[albroscani]

Thank you very much Vincent,
so I'll be waiting for the new release :)

Regards

Alberto

[vroyer]

Fixed in the release 2.1.1-9 (add full support of geo_point)