Freeze all dependencies for distribution upload

Commit f7e05dd earlier attempted to
freeze the dependency versions by adding version numbers to every
dependency we explicitly install for this project. However, that is not
sufficient. While that freezes the direct explicit dependencies, it does
not freeze the recursive dependencies (i.e., dependencies of
dependencies). The dependencies are free to install latest versions of
their dependencies which means our users may not get the exact recursive
dependencies that we have tested Cloudmarker with.

This commit is a better attempt to freeze all dependencies that exist in
this project's virtual Python environment. To achieve this, there are
now three requirement files.

  - `usr-requirements.txt`: These are our direct explicit dependencies,
  i.e., the dependencies we directly depend on for the project to run
  successfully from a user's perspective. Versions are generally not
  frozen here unless really required to work around an issue.

  - `dev-requirements.txt`: These are our direct explicit dependencies
  used during development phase only. Versions are not frozen here.
  Versions are generally not frozen here.

  - `pkg-requirements.txt`: These are the dependencies we package with
    our distribution. All versions are frozen here, so that a user who
    installs this project with `pip3` gets the exact versions of
    dependencies that we have tested this project with.

The `pkg-requirements.txt` file is updated with `make freeze` which
creates a new virtual Python environment, installs the requirements in
`usr-requirements.txt` and then writes the output of `pip3 freeze` to
this `pkg-requirements.txt`.

In this commit, the entry for `slackclient` in `usr-requirements.txt`
has been set to `slackclient==1.3.1` because that's the most recent
version of `slackclient` this project works with. This project does not
work fine with `slackclient 2.0.1` due to this error:

    ModuleNotFoundError: No module named 'slackclient'

In this commit, the entry for `urllib3` in `pkg-requirements.txt` has
been manually updated to `urllib3==1.24.3` to workaround this warning
that appears while installing dependencies:

    requests 2.21.0 has requirement urllib3<1.25,>=1.21.1, but you'll have
    urllib3 1.25.2 which is incompatible.

This issue also leads to an error when the project is installed from
source distribution with `pip3 install --no-binary :all: cloudmarker`
and run as `cloudmarker --help`.

    pkg_resources.ContextualVersionConflict:
    (urllib3 1.25.2 (venv/lib/python3.7/site-packages),
     Requirement.parse('urllib3<1.25,>=1.21.1'), {'requests'})

Setting the `urllib3` version manually resolves this issue.

MANIFEST.in

@@ -1,3 +1,3 @@
-# Installation from source distribution depends on requirements.txt
+# Installation from source distribution depends on pkg-requirements.txt
 # because setup.py reads this file.
-include requirements.txt
+include pkg-requirements.txt

Makefile

@@ -19,7 +19,7 @@ venv: FORCE
 # and modification times.
 deps: FORCE
 	touch venv
-	. ./venv && pip3 install -r requirements.txt
+	. ./venv && pip3 install -r usr-requirements.txt
 	. ./venv && pip3 install -r dev-requirements.txt
 
 rmvenv: FORCE
@@ -72,18 +72,9 @@ docs: FORCE
 checks: test coverage lint docs
 
 # Targets to build and upload a new release.
-freeze:
-	pkgs=$$(pip freeze); \
-	while read -r pkg; do \
-	    printf '%s' "$$pkgs" | grep -i "^$${pkg%==*}"; \
-	done < requirements.txt > requirements.tmp
-	mv requirements.tmp requirements.txt
-	cat requirements.txt
-
-unfreeze:
-	sed 's/\(.*\)==.*/\1/' requirements.txt > requirements.tmp
-	mv requirements.tmp requirements.txt
-	cat requirements.txt
+freeze: rmuservenv uservenv
+	. ./uservenv && pip3 install -r usr-requirements.txt
+	. ./uservenv && pip3 freeze > pkg-requirements.txt
 
 dist: clean
 	. ./venv && python3 setup.py sdist bdist_wheel
@@ -105,26 +96,34 @@ uservenv: FORCE
 
 rmuservenv: FORCE
 	rm -rf ~/.venv/cloudmarker-user
-	rm uservenv
+	rm -f uservenv
 
-install:
+verify-upload: verify-sdist verify-bdist
+
+verify-sdist:
 	@echo
 	@echo Testing source distribution from PyPI ...
 	@echo
 	make smoke-test PIP_OPTS="--no-binary :all:"
+
+verify-bdist:
 	@echo
 	@echo Testing wheel distribution from PyPI ...
 	@echo
 	make smoke-test
 
-test-install:
+verify-test-upload: verify-test-sdist verify-test-bdist
+
+verify-test-sdist:
 	@echo
 	@echo Testing source distribution from Test PyPI ...
 	@echo
 	make smoke-test PIP_OPTS="\
 	    --no-binary :all: \
 	    --index-url https://test.pypi.org/simple/ \
 	    --extra-index-url https://pypi.org/simple cloudmarker"
+
+verify-test-bdist:
 	@echo
 	@echo Testing wheel distribution from Test PyPI ...
 	@echo
@@ -133,9 +132,11 @@ test-install:
 	     --extra-index-url https://pypi.org/simple cloudmarker"
 
 smoke-test: rmuservenv uservenv
+	. ./uservenv && pip3 install -U pip
 	. ./uservenv && pip3 install $(PIP_OPTS) --no-cache-dir cloudmarker
 	. ./uservenv && cd /tmp && python3 -m cloudmarker --help
 	. ./uservenv && cd /tmp && cloudmarker --help
+	. ./uservenv && cd /tmp && cloudmarker --config --now
 
 clean: FORCE
 	find . -name "__pycache__" -exec rm -r {} +

cloudmarker/__init__.py

@@ -1,5 +1,5 @@
 """Cloudmarker - Cloud security monitoring framework."""
 
 
-__version__ = '0.1.0.dev23'
+__version__ = '0.1.0.dev29'
 __author__ = 'Cloudmarker Authors and Contributors'

pkg-requirements.txt

@@ -0,0 +1,38 @@
+adal==1.2.1
+asn1crypto==0.24.0
+azure-common==1.1.20
+azure-mgmt-compute==5.0.0
+azure-mgmt-network==2.7.0
+azure-mgmt-resource==2.1.0
+azure-mgmt-storage==3.3.0
+cachetools==3.1.0
+certifi==2019.3.9
+cffi==1.12.3
+chardet==3.0.4
+cryptography==2.6.1
+elasticsearch==7.0.1
+google-api-python-client==1.7.8
+google-auth==1.6.3
+google-auth-httplib2==0.0.3
+httplib2==0.12.3
+idna==2.8
+isodate==0.6.0
+msrest==0.6.6
+msrestazure==0.6.0
+oauthlib==3.0.1
+pyasn1==0.4.5
+pyasn1-modules==0.2.5
+pycparser==2.19
+PyJWT==1.7.1
+pymongo==3.8.0
+python-dateutil==2.8.0
+PyYAML==5.1
+requests==2.21.0
+requests-oauthlib==1.2.0
+rsa==4.0
+schedule==0.6.0
+six==1.12.0
+slackclient==1.3.1
+uritemplate==3.0.0
+urllib3==1.24.3
+websocket-client==0.54.0

setup.py

@@ -7,7 +7,7 @@
 _description = cloudmarker.__doc__.splitlines()[0]
 _long_description = open('README.rst').read()
 _version = cloudmarker.__version__
-_requires = open('requirements.txt').read().splitlines()
+_requires = open('pkg-requirements.txt').read().splitlines()
 
 setuptools.setup(
 

usr-requirements.txt

@@ -0,0 +1,10 @@
+azure-mgmt-compute
+azure-mgmt-network
+azure-mgmt-resource
+azure-mgmt-storage
+elasticsearch
+google-api-python-client
+pymongo
+PyYAML
+schedule
+slackclient==1.3.1