Skip to content

Commit

Permalink
Merge pull request yougov#11 from RiffynInc/vishram
Browse files Browse the repository at this point in the history
Mongo connector
  • Loading branch information
lauryl committed Aug 14, 2019
2 parents fc1d90e + b467d96 commit b814ea1
Show file tree
Hide file tree
Showing 8 changed files with 321 additions and 60 deletions.
7 changes: 6 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,16 @@ RUN pip install --upgrade pip

# Install mongo-connector
WORKDIR "/srv/riffyn/mongo-connector"

# the mongo-elastic.tar contains the mongo-connector and elastic2_doc_manager whls
ADD mongo-elastic.tar ./
RUN ls -l
# The order of installation is important. The commands overwrite previous commands.
# elastic2-doc-manager automatically installs stock-mongo connector.
# install stock elastic2-doc-manager because this brings in all the dependencies
RUN pip install elastic2-doc-manager[elastic5]
# then patch the modified elastic2_doc_manager to see our custom changes
RUN pip install elastic2-doc-manager/dist/elastic2_doc_manager-*.whl
# patch mongo_connector with our custom mongo_connector
RUN pip install mongo-connector/dist/mongo_connector-*.whl


Expand Down
42 changes: 42 additions & 0 deletions config/connector_property_types.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
{
"continueOnError": true,
"verbosity": 2,
"namespaces": {
"unity.propertyTypes": {
"rename": "property_types.propertyTypes",
"includeFields": [
"_id",
"immutable",
"name",
"definition",
"synonyms",
"source",
"created",
"accessibleTo",
"public",
"deleted"
]
}
},
"docManagers": [
{
"docManager": "elastic2_doc_manager",
"bulkSize": 10000,
"autoCommitInterval": 1,
"args": {
"clientOptions": {
"timeout": 100,
"sniffer_timeout": 60,
"sniff_on_start": true,
"sniff_on_connection_fail": true
}
}
}
],
"plugins": [
{
"pluginName": "UpdateElasticsearchIndex",
"moduleName": "mongo_connector.plugins.update_elasticsearch_index"
}
]
}
41 changes: 41 additions & 0 deletions config/connector_resource_types.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
{
"continueOnError": true,
"verbosity": 2,
"namespaces": {
"unity.resourceTypes": {
"rename": "resource_types.resourceTypes",
"includeFields": [
"_id",
"name",
"definition",
"synonyms",
"source",
"created",
"accessibleTo",
"public",
"deleted"
]
}
},
"docManagers": [
{
"docManager": "elastic2_doc_manager",
"bulkSize": 10000,
"autoCommitInterval": 1,
"args": {
"clientOptions": {
"timeout": 100,
"sniffer_timeout": 60,
"sniff_on_start": true,
"sniff_on_connection_fail": true
}
}
}
],
"plugins": [
{
"pluginName": "UpdateElasticsearchIndex",
"moduleName": "mongo_connector.plugins.update_elasticsearch_index"
}
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,6 @@
"continueOnError": true,
"verbosity": 2,
"namespaces": {
"unity.resourceTypes": {
"rename": "resource_types.resourceTypes",
"includeFields": [
"_id",
"name",
"definition",
"synonyms",
"source",
"created",
"accessibleTo",
"public",
"deleted"
]
},
"unity.propertyTypes": {
"rename": "property_types.propertyTypes",
"includeFields": [
"_id",
"immutable",
"name",
"definition",
"synonyms",
"source",
"created",
"accessibleTo",
"public",
"deleted"
]
},
"unity.resources": {
"rename": "resources_and_run_data.resources_and_run_data",
"includeFields": [
Expand Down Expand Up @@ -77,7 +48,6 @@
"docManagers": [
{
"docManager": "elastic2_doc_manager",
"targetURL": "elastic:9200",
"bulkSize": 10000,
"autoCommitInterval": 1,
"args": {
Expand All @@ -96,4 +66,4 @@
"moduleName": "mongo_connector.plugins.update_elasticsearch_index"
}
]
}
}
66 changes: 66 additions & 0 deletions config/mapping_property_types.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"dynamic": false,
"properties": {
"name": {
"type": "text",
"fields": {
"nostem": {
"type": "text",
"analyzer": "nonword",
"index_options": "docs"
},
"edge_ngram_property_type": {
"type": "text",
"analyzer": "autocomplete1",
"index_options": "docs"
}
}
},
"definition": {
"type": "text",
"analyzer": "english"
},
"synonyms": {
"type": "text",
"index_options": "docs",
"fields": {
"nostem": {
"type": "text",
"analyzer": "nonword",
"index_options": "docs"
},
"edge_ngram_property_type": {
"type": "text",
"analyzer": "autocomplete1",
"index_options": "docs"
}
}
},
"source": {
"type": "text",
"index": true
},
"created": {
"type": "date"
},
"version": {
"type": "text"
},
"immutable": {
"type": "boolean",
"index": true
},
"deleted": {
"type": "boolean",
"index": true
},
"accessibleTo": {
"properties": {
"_id": {
"type": "text",
"index": true
}
}
}
}
}
81 changes: 81 additions & 0 deletions config/mapping_resource_types.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
{
"dynamic": false,
"properties": {
"name": {
"type": "text",
"fields": {
"nostem": {
"type": "text",
"analyzer": "nonword",
"index_options": "docs"
},
"ngram_resource_type": {
"type": "text",
"analyzer": "entity_name",
"index_options": "docs"
},
"edge_ngram_resource_type": {
"type": "text",
"analyzer": "autocomplete1",
"index_options": "docs"
}
}
},
"definition": {
"type": "text",
"analyzer": "english"
},
"synonyms": {
"type": "text",
"index_options": "docs",
"fields": {
"nostem": {
"type": "text",
"analyzer": "nonword",
"index_options": "docs"
},
"ngram_resource_type": {
"type": "text",
"analyzer": "entity_name",
"index_options": "docs"
},
"edge_ngram_resource_type": {
"type": "text",
"analyzer": "autocomplete1",
"index_options": "docs"
}
}
},
"source": {
"type": "text",
"index": true
},
"created": {
"type": "date",
"index": true
},
"modified": {
"type": "date",
"index": true
},
"version": {
"type": "text"
},
"immutable": {
"type": "boolean",
"index": true
},
"deleted": {
"type": "boolean",
"index": true
},
"accessibleTo": {
"properties": {
"_id": {
"type": "keyword",
"index": true
}
}
}
}
}
Loading

0 comments on commit b814ea1

Please sign in to comment.