From 5d70e1291eee9284fe76c2dafe32e2e1c624fe01 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Tue, 30 Jul 2013 13:25:12 -0400 Subject: [PATCH] Fix for issue #105 --- resources/issues/105/01_create-river.bat | 4 +++ resources/issues/105/02_test-issue-105.bat | 7 +++++ resources/issues/105/custom-mapping.json | 19 ++++++++++++ resources/issues/105/disabled-mapping.json | 3 ++ resources/issues/105/drop-collection-105.js | 2 ++ .../issues/105/mongodb-river-simple.json | 14 +++++++++ resources/issues/105/test-issue-105.js | 10 +++++++ .../river/mongodb/MongoDBRiver.java | 30 +++++++++---------- 8 files changed, 74 insertions(+), 15 deletions(-) create mode 100644 resources/issues/105/01_create-river.bat create mode 100644 resources/issues/105/02_test-issue-105.bat create mode 100644 resources/issues/105/custom-mapping.json create mode 100644 resources/issues/105/disabled-mapping.json create mode 100644 resources/issues/105/drop-collection-105.js create mode 100644 resources/issues/105/mongodb-river-simple.json create mode 100644 resources/issues/105/test-issue-105.js diff --git a/resources/issues/105/01_create-river.bat b/resources/issues/105/01_create-river.bat new file mode 100644 index 00000000..39f901ff --- /dev/null +++ b/resources/issues/105/01_create-river.bat @@ -0,0 +1,4 @@ +curl -XPUT "http://localhost:9200/mydb105" -d @disabled-mapping.json +curl -XPUT "http://localhost:9200/mydb105/document/_mapping" -d @custom-mapping.json +pause +curl -XPUT "http://localhost:9200/_river/river105/_meta" -d @mongodb-river-simple.json \ No newline at end of file diff --git a/resources/issues/105/02_test-issue-105.bat b/resources/issues/105/02_test-issue-105.bat new file mode 100644 index 00000000..11929bc0 --- /dev/null +++ b/resources/issues/105/02_test-issue-105.bat @@ -0,0 +1,7 @@ +%MONGO_HOME%\bin\mongo < test-issue-105.js +pause +curl -XGET localhost:9200/mydb105/_search?pretty=true&q=firstName:John +pause +%MONGO_HOME%\bin\mongo < drop-collection-105.js +pause +curl -XGET localhost:9200/mydb105/_mapping?pretty=true \ No newline at end of file diff --git a/resources/issues/105/custom-mapping.json b/resources/issues/105/custom-mapping.json new file mode 100644 index 00000000..7815271d --- /dev/null +++ b/resources/issues/105/custom-mapping.json @@ -0,0 +1,19 @@ +{ + "document": { + "properties": { + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "nickName": { + "type": "string", "index":"not_analyzed" + }, + "creationDate": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} \ No newline at end of file diff --git a/resources/issues/105/disabled-mapping.json b/resources/issues/105/disabled-mapping.json new file mode 100644 index 00000000..4bb0ad98 --- /dev/null +++ b/resources/issues/105/disabled-mapping.json @@ -0,0 +1,3 @@ +{ + "index.mapper.dynamic": false +} \ No newline at end of file diff --git a/resources/issues/105/drop-collection-105.js b/resources/issues/105/drop-collection-105.js new file mode 100644 index 00000000..8bf96db8 --- /dev/null +++ b/resources/issues/105/drop-collection-105.js @@ -0,0 +1,2 @@ +use mydb105 +db.document.drop() \ No newline at end of file diff --git a/resources/issues/105/mongodb-river-simple.json b/resources/issues/105/mongodb-river-simple.json new file mode 100644 index 00000000..0e63969f --- /dev/null +++ b/resources/issues/105/mongodb-river-simple.json @@ -0,0 +1,14 @@ +{ + "type": "mongodb", + "mongodb": { + "db": "mydb105", + "collection": "document", + "options": { + "drop_collection": true + } + }, + "index": { + "name": "mydb105", + "type": "document" + } +} \ No newline at end of file diff --git a/resources/issues/105/test-issue-105.js b/resources/issues/105/test-issue-105.js new file mode 100644 index 00000000..5fd6d96d --- /dev/null +++ b/resources/issues/105/test-issue-105.js @@ -0,0 +1,10 @@ +use mydb105 +var o = +{ + "firstName": "John", + "lastName": "Doe", + "nickName": "The Boss", + "creationDate": "2013-06-11T12:00:00.0000-04:00" +} + +db.document.save(o) \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/river/mongodb/MongoDBRiver.java b/src/main/java/org/elasticsearch/river/mongodb/MongoDBRiver.java index 3e2f86e8..2dc9adb8 100644 --- a/src/main/java/org/elasticsearch/river/mongodb/MongoDBRiver.java +++ b/src/main/java/org/elasticsearch/river/mongodb/MongoDBRiver.java @@ -47,8 +47,7 @@ import org.bson.types.ObjectId; import org.elasticsearch.ElasticSearchInterruptedException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; @@ -884,6 +883,8 @@ private BSONTimestamp updateBulkRequest(final BulkRequestBuilder bulk, .admin().cluster().prepareState().execute() .actionGet().getState().getMetaData() .index(index).mappings(); + logger.trace("mappings contains type {}: {}", type, + mappings.containsKey(type)); if (mappings.containsKey(type)) { /* * Issue #105 - Mapping changing from custom @@ -893,16 +894,15 @@ private BSONTimestamp updateBulkRequest(final BulkRequestBuilder bulk, * before to delete. */ MappingMetaData mapping = mappings.get(type); - DeleteMappingRequest deleteMappingRequest = new DeleteMappingRequest( - index); - deleteMappingRequest.type(type); - client.admin().indices() - .deleteMapping(deleteMappingRequest); - PutMappingRequest putMapping = new PutMappingRequest( - index); - putMapping.type(type); - putMapping.source(mapping.getSourceAsMap()); - client.admin().indices().putMapping(putMapping); + client.admin().indices().prepareDeleteMapping(index).setType(type).execute().actionGet(); + PutMappingResponse pmr = client.admin() + .indices().preparePutMapping(index) + .setType(type) + .setSource(mapping.source().string()) + .execute().actionGet(); + if (!pmr.isAcknowledged()) { + logger.error("Failed to put mapping {} / {} / {}.", index, type, mapping.source()); + } } deletedDocuments = 0; @@ -1204,8 +1204,8 @@ private void processOplogEntry(final DBObject entry) throw new NullPointerException(MONGODB_ID_FIELD); } logger.info("Add attachment: {}", objectId); - object = MongoDBHelper.applyExcludeFields(object, - excludeFields); + object = MongoDBHelper + .applyExcludeFields(object, excludeFields); HashMap data = new HashMap(); data.put(IS_MONGODB_ATTACHMENT, true); data.put(MONGODB_ATTACHMENT, object); @@ -1324,7 +1324,7 @@ private void addQueryToStream(final String operation, "addQueryToStream - operation [{}], currentTimestamp [{}], update [{}]", operation, currentTimestamp, update); } - + for (DBObject item : slurpedCollection.find(update, findKeys)) { addToStream(operation, currentTimestamp, item.toMap()); }