refactor: Update ES index maps to use same maps of amundsen-common (#385

) * Update setup.py to remove python2.7 Signed-off-by: Wonyeong Choi <ciwnyg0815@gmail.com> * Update requirements.txt to include amundsen-common Signed-off-by: Wonyeong Choi <ciwnyg0815@gmail.com> * Replace ES index maps with amundsen-common's Signed-off-by: Wonyeong Choi <ciwnyg0815@gmail.com> * Remove ES constants file Signed-off-by: Wonyeong Choi <ciwnyg0815@gmail.com> * Remove ES constants file Signed-off-by: Wonyeong Choi <ciwnyg0815@gmail.com>
amundsen-io · Oct 16, 2020 · 20c2fd2 · 20c2fd2
1 parent 6cced36
commit 20c2fd2
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 253 deletions.
diff --git a/databuilder/publisher/elasticsearch_constants.py b/databuilder/publisher/elasticsearch_constants.py
diff --git a/databuilder/publisher/elasticsearch_publisher.py b/databuilder/publisher/elasticsearch_publisher.py
@@ -8,8 +8,9 @@
 from pyhocon import ConfigTree
 from typing import List
 
+from amundsen_common.models.index_map import TABLE_INDEX_MAP as TABLE_ELASTICSEARCH_INDEX_MAPPING
+
 from databuilder.publisher.base_publisher import Publisher
-from databuilder.publisher.elasticsearch_constants import TABLE_ELASTICSEARCH_INDEX_MAPPING
 
 LOGGER = logging.getLogger(__name__)
 

diff --git a/docs/dashboard_ingestion_guide.md b/docs/dashboard_ingestion_guide.md
@@ -111,7 +111,7 @@ job = DefaultJob(conf=job_config,
 job.launch()
 ```
 
-*Note that `DASHBOARD_ELASTICSEARCH_INDEX_MAPPING` is defined [here](../databuilder/publisher/elasticsearch_constants.py).  
+*Note that `DASHBOARD_ELASTICSEARCH_INDEX_MAPPING` is defined [here](https://github.com/amundsen-io/amundsencommon/blob/master/amundsen_common/models/index_map.py).  
 
 
 ### 4. Remove stale data

diff --git a/example/scripts/sample_data_loader.py b/example/scripts/sample_data_loader.py
@@ -29,15 +29,15 @@
 from elasticsearch import Elasticsearch
 from pyhocon import ConfigFactory
 from sqlalchemy.ext.declarative import declarative_base
+from amundsen_common.models.index_map import DASHBOARD_ELASTICSEARCH_INDEX_MAPPING
+from amundsen_common.models.index_map import USER_INDEX_MAP as USER_ELASTICSEARCH_INDEX_MAPPING
 
 from databuilder.extractor.csv_extractor import CsvTableColumnExtractor, CsvExtractor
 from databuilder.extractor.neo4j_es_last_updated_extractor import Neo4jEsLastUpdatedExtractor
 from databuilder.extractor.neo4j_search_data_extractor import Neo4jSearchDataExtractor
 from databuilder.job.job import DefaultJob
 from databuilder.loader.file_system_elasticsearch_json_loader import FSElasticsearchJSONLoader
 from databuilder.loader.file_system_neo4j_csv_loader import FsNeo4jCSVLoader
-from databuilder.publisher.elasticsearch_constants import DASHBOARD_ELASTICSEARCH_INDEX_MAPPING, \
-    USER_ELASTICSEARCH_INDEX_MAPPING
 from databuilder.publisher.elasticsearch_publisher import ElasticsearchPublisher
 from databuilder.publisher.neo4j_csv_publisher import Neo4jCsvPublisher
 from databuilder.task.task import DefaultTask

diff --git a/example/scripts/sample_tableau_data_loader.py b/example/scripts/sample_tableau_data_loader.py
@@ -23,12 +23,12 @@
 from elasticsearch import Elasticsearch
 from pyhocon import ConfigFactory
 from sqlalchemy.ext.declarative import declarative_base
+from amundsen_common.models.index_map import DASHBOARD_ELASTICSEARCH_INDEX_MAPPING
 
 from databuilder.extractor.neo4j_search_data_extractor import Neo4jSearchDataExtractor
 from databuilder.job.job import DefaultJob
 from databuilder.loader.file_system_elasticsearch_json_loader import FSElasticsearchJSONLoader
 from databuilder.loader.file_system_neo4j_csv_loader import FsNeo4jCSVLoader
-from databuilder.publisher.elasticsearch_constants import DASHBOARD_ELASTICSEARCH_INDEX_MAPPING
 from databuilder.publisher.elasticsearch_publisher import ElasticsearchPublisher
 from databuilder.publisher.neo4j_csv_publisher import Neo4jCsvPublisher
 from databuilder.task.task import DefaultTask

diff --git a/requirements.txt b/requirements.txt
@@ -38,6 +38,10 @@ typing==3.6.4
 # Upstream url: https://pypi.org/project/elasticsearch/
 elasticsearch>=6.2.0,<7.0
 
+# A common package that holds the models deifnition and schemas that are used
+# accross different amundsen repositories.
+amundsen-common>=0.5.6,<1.0
+
 atomicwrites==1.1.5
 more-itertools==4.2.0
 pluggy>=0.6.0

diff --git a/setup.py b/setup.py
@@ -70,7 +70,6 @@
     install_requires=requirements,
     python_requires='>=3.6,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*',
     extras_require={
-        ':python_version=="2.7"': ['typing>=3.6'],  # allow typehinting PY2
         'all': all_deps,
         'kafka': kafka,  # To use with Kafka source extractor
         'cassandra': cassandra,