Fix python packaging

- Use wheels instead of eggs
- Use requirements.txt to track dependencies instead of setup.py

.gitignore

@@ -1,7 +1,7 @@
-/.eggs/
-/cluster_mgr_eggs/
-/queue_mgr_eggs/
-/config_mgr_eggs/
+/.wheelhouse/
+/cluster_mgr_wheelhouse/
+/queue_mgr_wheelhouse/
+/config_mgr_wheelhouse/
 /build_clustermgr/
 /build_configmgr/
 /build_shared/

Makefile

@@ -70,59 +70,44 @@ coverage: ${ENV_DIR}/bin/coverage cluster_mgr_setup.py queue_mgr_setup.py config
 	${ENV_DIR}/bin/coverage xml
 
 .PHONY: env
-env: cluster_mgr_setup.py queue_mgr_setup.py config_mgr_setup.py shared_setup.py plugins_setup.py $(ENV_DIR)/bin/python build-eggs
+env: cluster_mgr_setup.py queue_mgr_setup.py config_mgr_setup.py shared_setup.py plugins_setup.py $(ENV_DIR)/bin/python build-wheelhouse
 
 $(ENV_DIR)/bin/python:
 	# Set up the virtual environment
 	virtualenv --setuptools --python=$(PYTHON_BIN) $(ENV_DIR)
 	$(ENV_DIR)/bin/easy_install "setuptools==24"
 	$(ENV_DIR)/bin/easy_install distribute
 
-BANDIT_EXCLUDE_LIST = src/metaswitch/clearwater/queue_manager/test/,src/metaswitch/clearwater/plugin_tests/,src/metaswitch/clearwater/etcd_tests/,src/metaswitch/clearwater/etcd_shared/test,src/metaswitch/clearwater/config_manager/test/,src/metaswitch/clearwater/cluster_manager/test/,common,_env,.eggs,debian,build_clustermgr,build_configmgr,build_shared
+BANDIT_EXCLUDE_LIST = src/metaswitch/clearwater/queue_manager/test/,src/metaswitch/clearwater/plugin_tests/,src/metaswitch/clearwater/etcd_tests/,src/metaswitch/clearwater/etcd_shared/test,src/metaswitch/clearwater/config_manager/test/,src/metaswitch/clearwater/cluster_manager/test/,common,_env,.wheelhouse,debian,build_clustermgr,build_configmgr,build_shared
 include build-infra/cw-deb.mk
 include build-infra/python.mk
 
-.PHONY: build-eggs
-build-eggs: ${ENV_DIR}/.cluster-mgr-build-eggs ${ENV_DIR}/.queue-mgr-build-eggs ${ENV_DIR}/.config-mgr-build-eggs
+.PHONY: build-wheelhouse
+build-wheelhouse: ${ENV_DIR}/.cluster-mgr-build-wheelhouse ${ENV_DIR}/.queue-mgr-build-wheelhouse ${ENV_DIR}/.config-mgr-build-wheelhouse
 
-${ENV_DIR}/.queue-mgr-build-eggs: queue_mgr_setup.py shared_setup.py common/setup.py $(shell find src/metaswitch -type f -not -name "*.pyc") $(shell find common/metaswitch -type f -not -name "*.pyc") src/metaswitch/clearwater/queue_manager/alarm_constants.py
-        # Generate .egg files
-	${ENV_DIR}/bin/python queue_mgr_setup.py build -b build_queuemgr bdist_egg -d queue_mgr_eggs
-	${ENV_DIR}/bin/python shared_setup.py build -b build_shared bdist_egg -d queue_mgr_eggs
-	cd common && EGG_DIR=../queue_mgr_eggs make build_common_egg
+define python_component
+# 1 Component Name - dash separated
 
-        # Download the egg files they depend upon
-	${ENV_DIR}/bin/easy_install -zmaxd queue_mgr_eggs/ queue_mgr_eggs/clearwater_queue_manager-1.0-py2.7.egg
-	${ENV_DIR}/bin/easy_install -zmaxd queue_mgr_eggs/ queue_mgr_eggs/clearwater_etcd_shared-1.0-py2.7.egg
-	${ENV_DIR}/bin/easy_install -zmaxd queue_mgr_eggs/ queue_mgr_eggs/metaswitchcommon-0.1-py2.7-linux-x86_64.egg
+build-wheelhouse: ${ENV_DIR}/.$1-build-wheelhouse
 
-	touch $@
+${ENV_DIR}/.$1-build-wheelhouse: $$(subst -,_,$1)_setup.py shared_setup.py common/setup.py $(shell find src/metaswitch -type f -not -name "*.pyc") $(shell find common/metaswitch -type f -not -name "*.pyc") src/metaswitch/clearwater/queue_manager/alarm_constants.py
 
-${ENV_DIR}/.config-mgr-build-eggs: config_mgr_setup.py shared_setup.py common/setup.py $(shell find src/metaswitch -type f -not -name "*.pyc") $(shell find common/metaswitch -type f -not -name "*.pyc") src/metaswitch/clearwater/config_manager/alarm_constants.py
-	# Generate .egg files
-	${ENV_DIR}/bin/python config_mgr_setup.py build -b build_configmgr bdist_egg -d config_mgr_eggs
-	${ENV_DIR}/bin/python shared_setup.py build -b build_shared bdist_egg -d config_mgr_eggs
-	cd common && EGG_DIR=../config_mgr_eggs make build_common_egg
+	rm -f $$@
 
-	# Download the egg files they depend upon
-	${ENV_DIR}/bin/easy_install -zmaxd config_mgr_eggs/ config_mgr_eggs/clearwater_config_manager-1.0-py2.7.egg
-	${ENV_DIR}/bin/easy_install -zmaxd config_mgr_eggs/ config_mgr_eggs/clearwater_etcd_shared-1.0-py2.7.egg
-	${ENV_DIR}/bin/easy_install -zmaxd config_mgr_eggs/ config_mgr_eggs/metaswitchcommon-0.1-py2.7-linux-x86_64.egg
+	# Generate wheels
+	${PYTHON} $$(subst -,_,$1)_setup.py build -b build_queuemgr bdist_wheel -d $$(subst -,_,$1)_wheelhouse
+	${PYTHON} shared_setup.py build -b build_shared bdist_wheel -d $$(subst -,_,$1)_wheelhouse
+	cd common && WHEELHOUSE=../$$(subst -,_,$1)_wheelhouse make build_common_wheel
 
-	touch $@
+	# Download the required dependencies
+	${PIP} wheel -w $$(subst -,_,$1)_wheelhouse -r $$(subst -,_,$1)-requirements.txt -r shared-requirements.txt -r common/requirements.txt --find-links $$(subst -,_,$1)_wheelhouse
 
-${ENV_DIR}/.cluster-mgr-build-eggs: cluster_mgr_setup.py shared_setup.py common/setup.py $(shell find src/metaswitch -type f -not -name "*.pyc") $(shell find common/metaswitch -type f -not -name "*.pyc") src/metaswitch/clearwater/cluster_manager/alarm_constants.py
-	# Generate .egg files
-	${ENV_DIR}/bin/python cluster_mgr_setup.py build -b build_clustermgr bdist_egg -d cluster_mgr_eggs
-	${ENV_DIR}/bin/python shared_setup.py build -b build_shared bdist_egg -d cluster_mgr_eggs
-	cd common && EGG_DIR=../cluster_mgr_eggs make build_common_egg
+	touch $$@
+endef
 
-	# Download the egg files they depend upon
-	${ENV_DIR}/bin/easy_install -zmaxd cluster_mgr_eggs/ cluster_mgr_eggs/clearwater_cluster_manager-1.0-py2.7.egg
-	${ENV_DIR}/bin/easy_install -zmaxd cluster_mgr_eggs/ cluster_mgr_eggs/clearwater_etcd_shared-1.0-py2.7.egg
-	${ENV_DIR}/bin/easy_install -zmaxd cluster_mgr_eggs/ cluster_mgr_eggs/metaswitchcommon-0.1-py2.7-linux-x86_64.egg
-
-	touch $@
+$(eval $(call python_component,queue-mgr))
+$(eval $(call python_component,config-mgr))
+$(eval $(call python_component,cluster-mgr))
 
 src/metaswitch/clearwater/queue_manager/alarm_constants.py: clearwater-queue-manager.root/usr/share/clearwater/infrastructure/alarms/clearwater_queue_manager_alarms.json common/metaswitch/common/alarms_writer.py common/metaswitch/common/alarms_parser.py common/metaswitch/common/alarm_severities.py
 	python common/metaswitch/common/alarms_writer.py --json-file="clearwater-queue-manager.root/usr/share/clearwater/infrastructure/alarms/clearwater_queue_manager_alarms.json" --constants-file=$@
@@ -132,7 +117,7 @@ src/metaswitch/clearwater/cluster_manager/alarm_constants.py: clearwater-cluster
 	python common/metaswitch/common/alarms_writer.py --json-file="clearwater-cluster-manager.root/usr/share/clearwater/infrastructure/alarms/clearwater_cluster_manager_alarms.json" --constants-file=$@
 
 .PHONY: deb
-deb: env build-eggs deb-only
+deb: env build-wheelhouse deb-only
 
 .PHONY: clean
 clean: envclean pyclean
@@ -147,7 +132,7 @@ pyclean:
 
 .PHONY: envclean
 envclean:
-	rm -rf bin cluster_mgr_eggs queue_mgr_eggs config_mgr_eggs develop-eggs parts .installed.cfg bootstrap.py .downloads .buildout_downloads *.egg .eggs *.egg-info
+	rm -rf bin cluster_mgr_wheelhouse queue_mgr_wheelhouse config_mgr_wheelhouse develop-wheelhouse parts .installed.cfg bootstrap.py .downloads .buildout_downloads *.egg .wheelhouse *.egg-info
 	rm -rf distribute-*.tar.gz
 	rm -rf $(ENV_DIR)
 

cluster_mgr_setup.py

@@ -27,8 +27,8 @@
         "clearwater_etcd_shared",
         "metaswitchcommon"],
     tests_require=[
-        "funcsigs==1.0.2",
-        "Mock==2.0.0",
-        "pbr==1.6",
-        "six==1.10.0"]
+        "funcsigs",
+        "Mock",
+        "pbr",
+        "six"]
     )

config_mgr_setup.py

@@ -27,8 +27,8 @@
         "clearwater_etcd_shared",
         "metaswitchcommon"],
     tests_require=[
-        "funcsigs==1.0.2",
-        "Mock==2.0.0",
-        "pbr==1.6",
-        "six==1.10.0"]
+        "funcsigs",
+        "Mock",
+        "pbr",
+        "six"]
     )

debian/clearwater-cluster-manager.install

@@ -1,2 +1,2 @@
-cluster_mgr_eggs/* usr/share/clearwater/clearwater-cluster-manager/eggs/
+cluster_mgr_wheelhouse/* usr/share/clearwater/clearwater-cluster-manager/wheelhouse/
 clearwater-cluster-manager.root/* /

debian/clearwater-cluster-manager.postinst

@@ -40,7 +40,13 @@ case "$1" in
 
         # Set up the virtual env
         virtualenv --python=$(which python) $BASE_DIR/env
-        $BASE_DIR/env/bin/easy_install --allow-hosts=None --always-unzip -f $BASE_DIR/eggs/ $BASE_DIR/eggs/clearwater_cluster_manager-1.0-py2.7.egg
+        $BASE_DIR/env/bin/pip install --compile \
+                    --no-index \
+                    --upgrade \
+                    --pre \
+                    --force-reinstall \
+                    --find-links $BASE_DIR/wheelhouse \
+                    clearwater_cluster_manager
         chown -R $NAME:root $BASE_DIR/env
 
         # Start monit monitoring

debian/clearwater-config-manager.install

@@ -1,4 +1,4 @@
-config_mgr_eggs/* usr/share/clearwater/clearwater-config-manager/eggs/
+config_mgr_wheelhouse/* usr/share/clearwater/clearwater-config-manager/wheelhouse/
 clearwater-config-manager.root/* /
 
 # Install the etcd plugins

debian/clearwater-config-manager.postinst

@@ -40,7 +40,13 @@ case "$1" in
 
         # Set up the virtual env
         virtualenv --python=$(which python) $BASE_DIR/env
-        $BASE_DIR/env/bin/easy_install --allow-hosts=None --always-unzip -f $BASE_DIR/eggs/ $BASE_DIR/eggs/clearwater_config_manager-1.0-py2.7.egg
+        $BASE_DIR/env/bin/pip install --compile \
+                    --no-index \
+                    --upgrade \
+                    --pre \
+                    --force-reinstall \
+                    --find-links $BASE_DIR/wheelhouse \
+                    clearwater_config_manager
         chown -R $NAME:root $BASE_DIR/env
 
         # Start monit monitoring

debian/clearwater-queue-manager.install

@@ -1,4 +1,4 @@
-queue_mgr_eggs/* usr/share/clearwater/clearwater-queue-manager/eggs/
+queue_mgr_wheelhouse/* usr/share/clearwater/clearwater-queue-manager/wheelhouse/
 clearwater-queue-manager.root/* /
 
 # Install the etcd plugin

debian/clearwater-queue-manager.postinst

@@ -40,7 +40,13 @@ case "$1" in
 
         # Set up the virtual env
         virtualenv --python=$(which python) $BASE_DIR/env
-        $BASE_DIR/env/bin/easy_install --allow-hosts=None --always-unzip -f $BASE_DIR/eggs/ $BASE_DIR/eggs/clearwater_queue_manager-1.0-py2.7.egg
+        $BASE_DIR/env/bin/pip install --compile \
+                    --no-index \
+                    --upgrade \
+                    --pre \
+                    --force-reinstall \
+                    --find-links $BASE_DIR/wheelhouse \
+                    clearwater_queue_manager
         chown -R $NAME:root $BASE_DIR/env
 
         # Start monit monitoring

queue_mgr_setup.py

@@ -27,8 +27,8 @@
         "clearwater_etcd_shared",
         "metaswitchcommon"],
     tests_require=[
-        "funcsigs==1.0.2",
-        "Mock==2.0.0",
-        "pbr==1.6",
-        "six==1.10.0"]
+        "funcsigs",
+        "Mock",
+        "pbr",
+        "six"]
     )

shared-requirements.txt

@@ -0,0 +1,10 @@
+docopt==0.6.2
+futures==3.0.5
+prctl==1.0.1
+# Note - if you are updating the version of python-etcd, check if you should
+# remove the monkeypatch in the common_etcd_synchronizer
+python-etcd==0.4.3
+py2_ipaddress==3.4.1
+pyyaml==3.11
+six==1.10.0
+urllib3==1.21.1

shared_setup.py

@@ -21,20 +21,18 @@
     package_data={
         '': ['*.eml'],
         },
-    # Note - if you are updating the version of python-etcd, check if you should
-    # remove the monkeypatch in the common_etcd_synchronizer
     install_requires=[
-        "docopt==0.6.2",
-        "futures==3.0.5",
-        "prctl==1.0.1",
-        "python-etcd==0.4.3",
-        "py2_ipaddress==3.4.1",
-        "pyyaml==3.11",
-        "six==1.10.0",
-        "urllib3==1.21.1"],
+        "docopt",
+        "futures",
+        "prctl",
+        "python-etcd",
+        "py2_ipaddress",
+        "pyyaml",
+        "six",
+        "urllib3"],
     tests_require=[
-        "funcsigs==1.0.2",
-        "Mock==2.0.0",
-        "pbr==1.6",
-        "six==1.10.0"],
+        "funcsigs",
+        "Mock",
+        "pbr",
+        "six"],
     )