Cap explicit dependencies

pin all global-requirements with the version found in pip-freeze
along with a tool to generate the new requirements file

Issues:
* Some versions are only apt-get installable (suds 0.4.1)
* Not all packages are installed in our standard dsvm-tempest env
* Some versions are lower then the minimum requirement we previously had
  because python is awful (libvirt-python)
* Doesn't pin transitive dependencies
* library versions can vanish from pypi and still break us

Using pip-freeze file from a recently stable/juno neutron-full patch:
http://logs.openstack.org/40/154740/1/check/check-tempest-dsvm-neutron-full/c232cfe/logs/pip-freeze.txt.gz
(saved as pip-freezes.txt here for record keeping)

Grenade and tempest-full pip-freezes are different and do not work with
each other, so some tweaking is required.

Taking the installed library version from pip-freeze and use that as a
version cap.

Change-Id: Iaf48bb069fdd7a19d614ce44b86abd9977c5f0c0

cap.py

@@ -0,0 +1,133 @@
+#! /usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import argparse
+import re
+
+import pkg_resources
+
+overrides = dict()
+# suds 0.4.1 isn't pip installable but is in distribution packages
+overrides['suds'] = 'suds==0.4'
+# apt package of libvirt-python is lower then our minimum requirement
+overrides['libvirt-python'] = None
+# Possible issue, use kombu 3.0.7 vs 3.0.24 ? grenade uses 3.0.7 tempest 3.0.24
+overrides['kombu'] = 'kombu==3.0.7'  # grenade version.
+
+
+def cap(requirements, frozen):
+    """Cap requirements to version in freeze.
+
+    Go through every package in requirements and try to cap.
+
+    Input: two arrays of lines.
+    Output: Array of new lines.
+    """
+    output = []
+    for line in requirements:
+        try:
+            req = pkg_resources.Requirement.parse(line)
+            specifier = str(req.specifier)
+            if any(op in specifier for op in ['==', '~=', '<']):
+                # if already capped, continue
+                output.append(line)
+                continue
+        except ValueError:
+            # line was a comment, continue
+            output.append(line)
+            continue
+        if req.project_name in overrides:
+            new_line = overrides[req.project_name]
+            if new_line:
+                output.append(overrides[req.project_name])
+            else:
+                output.append(line)
+            continue
+        # add cap
+        new_cap = cap_requirement(req.project_name, frozen)
+        if new_cap:
+            output.append(pin(line, new_cap))
+        else:
+            output.append(line)
+    return output
+
+
+def pin(line, new_cap):
+    """Add new cap into existing line
+
+    Don't use pkg_resources so we can preserve the comments.
+    """
+    end = None
+    use_comma = False
+    parts = line.split(' #')
+    if len(split(parts[0].strip())) > 1:
+        use_comma = True
+    if "#" in line:
+        # if comment
+        end = parts[1]
+    # cap to new max version
+    if end:
+        new_end = "<=%s #%s" % (new_cap, end)
+    else:
+        new_end = "<=%s" % new_cap
+    if use_comma is True:
+        return "%s,%s" % (parts[0].strip(), new_end)
+    else:
+        return "%s%s" % (parts[0].strip(), new_end)
+
+
+def split(line):
+    return re.split('[><=]', line)
+
+
+def cap_requirement(requirement, frozen):
+    # Find current version of requirement in freeze
+    specifier = frozen.get(requirement, None)
+    if specifier:
+        return split(str(specifier))[-1]
+    return None
+
+
+def freeze(lines):
+    """Parse lines from freeze file into a dict.
+
+    Where k:v is project_name:specifier.
+    """
+    freeze = dict()
+
+    for line in lines:
+        try:
+            req = pkg_resources.Requirement.parse(line)
+            freeze[req.project_name] = req.specifier
+        except ValueError:
+            # not a valid requirement, can be a comment, blank line etc
+            continue
+    return freeze
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('requirements', help='requirements file input')
+    parser.add_argument('freeze', help='output of pip freeze')
+    args = parser.parse_args()
+    with open(args.requirements) as f:
+        requirements = [line.strip() for line in f.readlines()]
+    with open(args.freeze) as f:
+        frozen = freeze([line.strip() for line in f.readlines()])
+    for line in cap(requirements, frozen):
+        print(line)
+
+if __name__ == '__main__':
+    main()