Merge branch 'master' of github.com:abrarsheikh/data_pipeline into pr…

…-2-abrar-internal_push

.travis.yml

@@ -3,20 +3,11 @@ branches:
   - master
   - /^v[0-9.]+$/
 language: python
+services:
+  - docker
 python:
-- '2.7'
-env:
-  matrix:
-  - MAKE_TARGET=test
+  - '2.7'
 install: pip install coveralls tox
-script: make -f Makefile-opensource "$MAKE_TARGET"
+script: make -f Makefile-opensource test
 after_success:
   - coveralls
-deploy:
-  provider: pypi
-  user: <TODO>
-  password:
-    secure: <TODO>
-  on:
-    tags: true
-    repo: Yelp/data_pipeline

Makefile-opensource

@@ -41,8 +41,7 @@ clean-vim:
 	find . -name '*.swo' -exec rm -f {} +
 
 test:
-	# This will timeout after 15 minutes, in case there is a hang on jenkins
-	OPEN_SOURCE_MODE=true PULL_CONTAINERS=true FORCE_FRESH_CONTAINERS=true timeout -9 1800 tox -c tox-opensource.ini $(REBUILD_FLAG)
+	OPEN_SOURCE_MODE=true PULL_CONTAINERS=true FORCE_FRESH_CONTAINERS=true tox -c tox-opensource.ini $(REBUILD_FLAG)
 
 docs: clean-docs 
 	tox -c tox-opensource.ini -e docs $(REBUILD_FLAG)

data_pipeline/config.py

@@ -228,7 +228,7 @@ def kafka_zookeeper(self):
     def zookeeper_discovery_path(self):
         return data_pipeline_conf.read_string(
             'zookeeper_discovery_path',
-            default='/nail/etc/zookeeper_discovery/generic/uswest2{ecosystem}.yaml'
+            default='zookeeper_discovery{ecosystem}.yaml'
         )
 
     @property
@@ -301,10 +301,11 @@ def encryption_type(self):
 
     @property
     def key_location(self):
-        """Directory in which to look for key to encrypt pii."""
+        """Directory in which to look for key to encrypt pii. ./key-1.key is meant for
+        unit tests and documentation purpose, please use a different key in production."""
         return data_pipeline_conf.read_string(
             'key_location',
-            default='/nail/srv/configs/data_pipeline/'
+            default='./'
         )
 
     @property
@@ -314,7 +315,7 @@ def data_pipeline_teams_config_file_path(self):
         """
         return data_pipeline_conf.read_string(
             'data_pipeline_teams_config_file_path',
-            default='/nail/etc/services/data_pipeline/teams.yaml'
+            default='teams.yaml'
         )
 
     @property
@@ -532,6 +533,11 @@ def max_producer_delay_seconds(self):
         greater than this time will produce an alert"""
         return data_pipeline_conf.read_int('sensu_max_delay_seconds', 300)
 
+    @property
+    def ecosystem_file_path(self):
+        """Path to the file that specifies the ecosystem."""
+        return data_pipeline_conf.read_string('ecosystem_file_path', "ecosystem")
+
 
 def configure_from_dict(config_dict):
     """Configure the :mod:`data_pipeline` clientlib from a dictionary.

data_pipeline/environment_configs.py

@@ -18,4 +18,4 @@
 
 import os
 
-IS_OPEN_SOURCE_MODE = os.getenv('OPEN_SOURCE_MODE', 'false').lower() in ['t', 'true', 'y', 'yes']
+IS_OPEN_SOURCE_MODE = os.getenv('OPEN_SOURCE_MODE', 'true').lower() in ['t', 'true', 'y', 'yes']

data_pipeline/testing_helpers/containers.py

@@ -36,7 +36,15 @@
 
 class ContainerUnavailableError(Exception):
     def __init__(self, project='unknown', service='unknown'):
-        Exception.__init__(self, "Container for project {0} and service {1} failed to start".format(project, service))
+        try:
+            msg = open("logs/docker-compose.log", 'r').read()
+        except IOError:
+            msg = "no error log."
+        Exception.__init__(self, "Container for project {0} and service {1} failed to start with {2}".format(
+            project,
+            service,
+            msg
+        ))
 
 
 class Containers(object):
@@ -164,7 +172,7 @@ def get_container_info(cls, project, service, docker_client=None):
             raise ContainerUnavailableError(project=project, service=service)
 
     @classmethod
-    def get_container_ip_address(cls, project, service, timeout_seconds=60):
+    def get_container_ip_address(cls, project, service, timeout_seconds=100):
         """Fetches the ip address assigned to the running container.
         Throws ContainerUnavailableError if the retry limit is reached.
 
@@ -418,4 +426,4 @@ def _run_compose(self, *args):
             )
 
     def _is_envvar_set(self, envvar):
-        return os.getenv(envvar, 'false').lower() in ['t', 'true', 'y', 'yes']
+        return os.getenv(envvar, 'true').lower() in ['t', 'true', 'y', 'yes']

data_pipeline/testing_helpers/docker-compose-opensource.yml

@@ -11,11 +11,11 @@ kafka:
 # TODO (DATAPIPE-1858|abrar): change all the yelpcorp url's to docker hub once we
 # push our docker images to docker hub.
 schematizerdatabase:
-  image: docker-dev.yelpcorp.com/schematizer_database:latest
+  image: docker.io/yelp/schematizer_database:latest
 schematizerconfigs:
-  image: docker-dev.yelpcorp.com/schematizer_configs:latest
+  image: docker.io/yelp/schematizer_configs:latest
 schematizer:
-  image: docker-dev.yelpcorp.com/schematizer_service:latestopensource
+  image: docker.io/yelp/schematizer_service:latest
   links:
     - schematizerdatabase
   volumes_from:

data_pipeline/tools/bootstrapper.py

@@ -383,6 +383,7 @@ def get_source_from_sql_file_path(self, sql_file_path):
 def is_file_bootstrapper_class(obj):
     return inspect.isclass(obj) and FileBootstrapperBase in obj.__bases__
 
+
 FILE_BOOTSTRAPPER_CLASSES = [
     obj for _, obj in inspect.getmembers(sys.modules[__name__], is_file_bootstrapper_class)
 ]

data_pipeline/tools/compaction_setter.py

@@ -148,5 +148,6 @@ def run(self):
         self.log.info("Found {} topics to compact".format(len(topics)))
         self.apply_log_compaction(topics)
 
+
 if __name__ == '__main__':
     CompactionSetter().start()

data_pipeline/tools/redshift_sql_to_avsc.py

@@ -465,5 +465,6 @@ def convert_sql_to_avsc(self, avsc_file_path, sql_file_path):
                 sort_keys=True
             )
 
+
 if __name__ == "__main__":
     RedshiftSQLToAVSCBatch().start()

data_pipeline/zookeeper.py

@@ -44,7 +44,7 @@ def max_tries(self):
 
     @cached_property
     def ecosystem(self):
-        return open('/nail/etc/ecosystem').read().strip()
+        return open(get_config().ecosystem_file_path).read().strip()
 
     def __init__(self):
         retry_policy = KazooRetry(max_tries=self.max_tries)

ecosystem

@@ -0,0 +1 @@
+devc

key-1.key

@@ -0,0 +1,49 @@
+MIIJKQIBAAKCAgEAgtOJ7j/3h+fynqWCV0C9EV0PMl7x6ht/QX9Ool6aGXuG7IOb
++PHhmpcPrSmoRaC2m3dJooLxzxAtIJ1JbNb9c55lvCg2ft/3d0CwwD/HMmCSQ3V7
+QykcgDgqW3W9RWlUa6mb4ExxFowwx5TLa0snCVhPv8FjwLC5z7NqispjuSVdwgoQ
+nY4fg4wpvH/ziHT2qkNBLP/y1xHmhzsaWWLxdFT4IBmHCWorQEmfB9WHwxEO510u
+cjkOjSZ66ba6yLgsgY1VMsCYj/rQpQeZYgjk7ZwelGJyvz/lrH87piSRE70zReAj
+GuGKEULYVPxsFT7xEhgwDbH6+8ZKo1Sz49DTgB9MtSOtI5UkSct1/Hv3SJoZXmDS
+Avn0MlyOxAYhlX9FVS2placyksM6lPfRHUH9SndOK4OOLVhnLvKyY/F9pFXx6QP1
+Ni0J+s5ylNBNl8dXbPBRtFRFgHnyZ85XDMQ0tDWULydkQEqbfuy0SWIgfS9ClBPo
+1nQNTNvlUp6re0CSz+I0siQ/ctl6F172/8m3TbNVf6FL7VKte6gksVWjeqDH66ww
+EyEQC7dLSGFiD/CFv/6k6zv5VZRrpjBISjvXg89h5BhWQ8bXqlbPdEGN0vgx9Kwm
+DV8eW3uQZenbepNdLOVIkDzKQZZ+q5C7FNiFM4dyzXQ3uMPcOw+p9nTXUwkCAwEA
+AQKCAgBbn174FqB68/y9Etjn1pUTNgNQmaH+/ObwrgIaQPrXYeWqermfbXc81xLJ
+iWUp7lY1JyPQxrgm9gNE1iL4MNEsSgCh99/Cl4QuZoXi/ke5hlypFfwPmxk0yyEP
+ELRPimbvR8e9HIwO6Hu5KwIT0Hw0VT704rHD34Eib1rsfLW+kubI4FemoER90ong
+VBPwu0mNELDynQYrjYPPrZrUgzDq8h8cDATAmTRVP7o6uFbuTGINsvq2UDNas1pd
+l/XUYPf6e0rx0mSGYXgCPAI14FjD+BSKgzAttMhbOHhD8VQRaO4hv8mWJ+KUGCZ9
+zpK7hms66umsjbti3QqkvMuHf1a/V9NwJ6Dsz41vAzqL4tGvbBrCd24nXeTMbMi8
+0QDhzAMt+z2sB2E087pLyt4AOS9AGSVbR2jBzac0Fkw3J+MAxcKWLbMDiI6NxUoC
+olHFDsISpefxqYDkP/+tO4fQbyzwpeArQ3MhmYwpkGfKez8zIFZPzxwSdz6KvEso
+HniYA2uR8muXshC+CV0H2aERa8Fuw88RFdHqAB7uS8wiNWTGy1XhByvVBwtgSu2R
+usO7r6C3+RMNcj3+i1tBFMLVatzn6Vqo+z2jsJ3NtWExl1Q0UsIeeEu8iwURmopv
+87INvTOoUssX1lS7fTxSCBmmMw6JN5kEGhL92CR4/E7DxEAMyQKCAQEA8cuAJ//e
+U8TFbCS6hr30qYDOx0vWZe2Ju1uVz6Sk5sKiKw5PuLnya46SIOY1zGVpT2IJCjut
+D2b+IgPateO50kClen7/DM49sBYp7KuFz3XZ7Oc4hq470mQ/wwmMc2nqY5DU4+wm
+6LL0quGK6nBIWE52MYqXFdCXc23ymv5eb4J0qzu5OwLzSAGZtMBXyCH9sKtzcGHJ
+E8eOfYNhE8O0/O2dpjpOJJTBuPhQztPKEBLJ5mnlOKoFmzQbJJBo4uwufjkIzD/p
+z1P4T/YJheroJMrZOAAVDUZpYDcPK1wtleYINvPUvdru6lxA10wHPysBLHsnSlvu
+PJRDTgk+x1XuZwKCAQEAioMdVuAf3ylgvTDNEQCxFjAW7uQkDi9zjkNblRXn/Xh1
+dvqtH0SnDXuq0ZkE5Ce1Bh03DQnw7YL21VfL6B10Q+7c7hu04WsI6zoX8AB9s7Si
+Jqm5hD4YWjSshwH7Hgojufr1IgYTiIpAGCmJlhOnFRRxj/oLAnJgUqtx8Vr/JNG5
+OwZB6ZRmhjhlzODvIIAo0LwRw4KiPj729OpkGAaL3eaHYJdLlGP0xClr60acEK6I
+OoEyoFr8ywNsewOOUjO522y75V32YhrbeQdQpjVdQuVuxPSJaSj2VjnKDewLZASr
+7HrNLhQ5qaXcwuUpkn+7eVSYey9Pcna8Mc467H/tDwKCAQEA5vDKV+n/7Jx9P5wY
+98cInrjjRuD9Rt9I9MU14lxMEAeIIn2J0VLw4qAKnBek/LNDkUl+fuKemZ5yTgTG
+aYG3IsUSCFyxuAZoTv3sTxtIM+1JFweFKsXRZ1LOv7coCbf44d4cKBGO3CTIoUnI
+h8B0qLtQEmNfiLtxXBOAa19WzUSZsE7bzSw1NuooZ8ZqUUF977biDE2pYxXv15ub
+Jeh/vjApqfH4XuIh4Uxyygjo7s8zPbqY77Uco69x2UuAknSot3A8IfFNnPODsMO7
+Nsg1r6Z7MAyMDuDYlsgf2fcxpwFNiUb8Q9WO6hLw73GzXjbx6Q07ar9BXWTyn98e
+sf82MwKCAQBj8FJc+n1EkSBhoTJ00CD1t2D0oNt6+LXRsbwEh7fJZP6sCBaddvVp
+fw8zB8tFq3irnqy31bJmTJY4PI69PXNECLaTfp41/vqCbnoTCuenn+9XIiLezcbY
+XyUUjFNZy0sXx4DTObsiS04PoPjKtgdZ6FQ/49PJwijJvTYLRPLO6BtUASHRnOeU
+dGDPQISI2K+aW/gdLsZNTzT0ZaIuy5pjUw/em4tG2BAk8RYSvfGg/6z/OXUDymKX
+QDMnVtt7aCLztTKlPfSluGceni6MnfhaahxQM3hpzcpfUHb/MI7Pbad2iuw8EQ/B
+3hC49ovWTh9AXc9fIjWaix0ieoB9X5f1AoIBAQCp9ALAlnj4rra9SRou7uxCLYSW
+IYeqLEkBhnOJJHHekEwFimRwRm9LxSMoYHtbuZByuZBo7bwp2wUj6wrtn1xzsB/r
+3KYX0Z+hDksLhBUQpTeiVAej6BbQUaOiNJ0M++EskpcM9J34nNBbWxk/zB/bFrRA
+MC7GGGQ6lXlj/e8YWHfUuLijGF7q9FUfLRAcCDwkoDdfnO/fEKRcKrDBc8ezGTa5
+WNLlA7a4nhw9JZ+rTK9wL4PH4GXxSTVkhk/NP/YMSwriGVfeLnWBpW+T/HNarHIk
+3j2M4/b72cIAtQtZfwRR8WZ5JXs8ZIn5UN4kM0MvoVTDdYh0ieLevuIfqA08

requirements.d/dev-internal.txt

@@ -19,4 +19,3 @@ terminaltables>=1.0.2
 wheel>=0.22
 factory_boy==2.5.2
 pytest-benchmark==3.0.0
-clog

requirements.d/dev.txt

@@ -17,4 +17,3 @@ SQLAlchemy==1.0.8
 terminaltables>=1.0.2
 wheel>=0.22
 factory_boy==2.5.2
-clog

setup.py

@@ -57,7 +57,8 @@
         'PyStaticConfiguration>=0.9.0',
         'simplejson>=2.1.2',
         'swagger_zipkin>=0.1.0',
-        'yelp-avro==1.9.2'
+        'yelp-avro==1.9.2',
+        'yelp-clog==2.2.11'
     ],
     extras_require={
         'tools': [

teams.yaml

@@ -0,0 +1,6 @@
+---
+teams:
+  bam:
+    email: team1@yelp.com
+  team2:
+    email: team2@yelp.com

tests/config_test.py

@@ -88,7 +88,7 @@ def test_active_encryption_key(self, config):
         assert config.encryption_type is None
 
     def test_data_pipeline_teams_config_file_path(self, config):
-        assert config.data_pipeline_teams_config_file_path == '/nail/etc/services/data_pipeline/teams.yaml'
+        assert config.data_pipeline_teams_config_file_path == 'teams.yaml'
 
     def test_kafka_client_ack_count(self, config):
         assert config.kafka_client_ack_count == -1
@@ -124,6 +124,8 @@ def setup_class(cls):
         """SmartStack needs to be configured to use service discovery, so
         doing that for the entire class.
         """
+        if IS_OPEN_SOURCE_MODE:
+            return
         staticconf.YamlConfiguration(
             '/nail/etc/services/services.yaml',
             namespace='smartstack_services'

tests/conftest.py

@@ -344,7 +344,7 @@ def kafka_docker(containers):
 def configure_teams():
     config_path = os.path.join(
         os.path.dirname(__file__),
-        'config/teams.yaml'
+        '../teams.yaml'
     )
     with reconfigure(data_pipeline_teams_config_file_path=config_path):
         yield

tests/tools/introspector/schema_check_command_test.py

@@ -30,6 +30,7 @@
 class FakeParserError(Exception):
     pass
 
+
 Args = namedtuple(
     "Namespace", [
         "schema",

tests/zookeeper_test.py

@@ -23,6 +23,7 @@
 from kazoo.client import KazooClient
 from kazoo.exceptions import LockTimeout
 
+from data_pipeline.environment_configs import IS_OPEN_SOURCE_MODE
 from data_pipeline.zookeeper import ZK
 from data_pipeline.zookeeper import ZKLock
 
@@ -121,10 +122,16 @@ def test_lock_exception(
         assert locked_zk_client.stop.call_count >= 1
         assert locked_zk_client.close.call_count >= 1
 
+    @pytest.mark.skipif(
+        IS_OPEN_SOURCE_MODE,
+        reason="This test is acquiring lock on local ZK instance and not on the mock."
+    )
     def test_double_lock(
         self,
         patch_exit
     ):
+        # TODO (DATAPIPE-2122|abrar): this test is aqcuiring
+        # lock on actual zookeeper instance, needs to be fixed.
         with ZKLock(self.fake_name, self.fake_namespace):
             assert patch_exit.call_count == 0
             with ZKLock(self.fake_name, self.fake_namespace):

tox-opensource.ini

@@ -3,14 +3,10 @@
 # py34, py35, pypy
 envlist = py27, docs
 skipsdist = true
-indexserver =
-    default = https://pypi.yelpcorp.com/simple/
 
 [testenv]
 basepython = python2.7
 envdir = venv/py27
-setenv =
-    PIP_INDEX_URL = https://pypi.yelpcorp.com/simple
 venv_update = {toxinidir}/bin/venv-update venv= {envdir} install= 
 commands =
 	{[testenv]venv_update} -r {toxinidir}/requirements.d/dev.txt

travis.yml

@@ -0,0 +1,11 @@
+branches:
+  only:
+    - master
+    - "/^v[0-9.]+$/"
+language: python
+python:
+  - '2.7'
+install: pip install coveralls tox
+script: make -f Makefile-opensource test
+after_success:
+  - coveralls

zookeeper_discoverydevc.yaml

@@ -0,0 +1,3 @@
+---
+- - zk
+  - 2181