Merge pull request #157 from mozbhearsum/allow-unsafe

Make it possible to forward --allow-unsafe to pip-compile

docs/features.rst

@@ -128,6 +128,22 @@ Example output:
 ``pip-compile-multi`` will recursively propagate this option to all environments
 that are referencing or referenced by selected environment name.
 
+Allow Unsafe Packages
+=====================
+If your project depends on packages that include ``setuptools`` or other packages
+considered "unsafe" by ``pip-tools`` and you still wish to have them included in
+the resulting requirements files, you can pass this option to do so:
+
+.. code-block:: text
+
+    --allow-unsafe / --no-allow-unsafe
+                                    Whether or not to include 'unsafe' packages
+                                    in generated requirements files. Consult
+                                    pip-compile --help for more information
+
+This is commonly used with --generate-hashes to avoid generating requirements files
+which cannot be installed.
+
 Custom Header
 =============
 

pipcompilemulti/cli_v1.py

@@ -22,6 +22,10 @@
               help='Environment name (base, test, etc) that needs '
                    'packages hashes. '
                    'Can be supplied multiple times.')
+@click.option('--allow-unsafe/--no-allow-unsafe', default=False,
+              help="Whether or not to include 'unsafe' packages "
+                   'in generated requirements files. '
+                   'Consult pip-compile --help for more information')
 @click.option('--directory', '-d', default=OPTIONS['base_dir'],
               help='Directory path with requirements files.')
 @click.option('--in-ext', '-i', default=OPTIONS['in_ext'],
@@ -39,7 +43,7 @@
               help='Only upgrade named package. Can be supplied multiple times.')
 @click.option('--use-cache', '-u', default=OPTIONS['use_cache'], is_flag=True,
               help='Use pip-tools cache to speed up compilation.')
-def cli(ctx, compatible, forbid_post, generate_hashes, directory,
+def cli(ctx, compatible, forbid_post, generate_hashes, allow_unsafe, directory,
         in_ext, out_ext, header, only_name, upgrade, upgrade_package, use_cache):
     """Recompile"""
 
@@ -52,6 +56,7 @@ def cli(ctx, compatible, forbid_post, generate_hashes, directory,
         'compatible_patterns': compatible,
         'forbid_post': set(forbid_post),
         'add_hashes': set(generate_hashes),
+        'allow_unsafe': allow_unsafe,
         'base_dir': directory,
         'in_ext': in_ext,
         'out_ext': out_ext,

pipcompilemulti/environment.py

@@ -133,6 +133,8 @@ def pin_command(self):
             parts.append('--rebuild')
         if self.add_hashes:
             parts.append('--generate-hashes')
+        if OPTIONS['allow_unsafe']:
+            parts.append('--allow-unsafe')
         parts.extend(['--output-file', self.outfile, self.infile])
         return parts
 

pipcompilemulti/options.py

@@ -4,6 +4,7 @@
 
 OPTIONS = {
     'add_hashes': [],
+    'allow_unsafe': False,
     'base_dir': 'requirements',
     'compatible_patterns': [],
     'forbid_post': [],

requirements/local.hash

@@ -1,4 +1,4 @@
-# SHA1:b35f2fa5ae43afc446aafff1deed3615ec4bab17
+# SHA1:b5c067a49051e2a7998840727390905f5fa5372f
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -18,9 +18,9 @@ bleach==3.1.0 \
 bump2version==0.5.11 \
     --hash=sha256:524bde030318fe2543038defe0f77739605636fef96924883813cb290cf79c1e \
     --hash=sha256:bfcc051498dda9fd9ac8634689f4516e1c20fdeeace3278932cc6e1248418b36
-certifi==2019.6.16 \
-    --hash=sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939 \
-    --hash=sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695
+certifi==2019.9.11 \
+    --hash=sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50 \
+    --hash=sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef
 cfgv==2.0.1 \
     --hash=sha256:edb387943b665bf9c434f717bf630fa78aecd53d5900d2e05da6ad6048553144 \
     --hash=sha256:fbd93c9ab0a523bf7daec408f3be2ed99a980e20b2d19b50fc184ca6b820d289
@@ -143,26 +143,26 @@ requirements-detector==0.6 \
     --hash=sha256:9fbc4b24e8b7c3663aff32e3eba34596848c6b91bd425079b386973bd8d08931
 setoptconf==0.2.0 \
     --hash=sha256:5b0b5d8e0077713f5d5152d4f63be6f048d9a1bb66be15d089a11c898c3cf49c
-snowballstemmer==1.9.0 \
-    --hash=sha256:9f3b9ffe0809d174f7047e121431acf99c89a7040f0ca84f94ba53a498e6d0c9
+snowballstemmer==1.9.1 \
+    --hash=sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e
 toml==0.10.0 \
     --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
     --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e
-tox==3.13.2 \
-    --hash=sha256:dab0b0160dd187b654fc33d690ee1d7bf328bd5b8dc6ef3bb3cc468969c659ba \
-    --hash=sha256:ee35ffce74933a6c6ac10c9a0182e41763140a5a5070e21b114feca56eaccdcd
+tox==3.14.0 \
+    --hash=sha256:0bc216b6a2e6afe764476b4a07edf2c1dab99ed82bb146a1130b2e828f5bff5e \
+    --hash=sha256:c4f6b319c20ba4913dbfe71ebfd14ff95d1853c4231493608182f66e566ecfe1
 tqdm==4.35.0 \
     --hash=sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9 \
     --hash=sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457
-twine==1.13.0 \
-    --hash=sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446 \
-    --hash=sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc
+twine==1.14.0 \
+    --hash=sha256:b2cec0dc1ac55bd74280d257f43763cf0cf928bdcd0de0fd70be70aa1195e3b0 \
+    --hash=sha256:e37d5a73d77b095b85314dde807bfb85b580b5b9d137f5b21332f4636990d97a
 urllib3==1.25.3 \
     --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
     --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232
-virtualenv==16.7.4 \
-    --hash=sha256:94a6898293d07f84a98add34c4df900f8ec64a570292279f6d91c781d37fd305 \
-    --hash=sha256:f6fc312c031f2d2344f885de114f1cb029dfcffd26aa6e57d2ee2296935c4e7d
+virtualenv==16.7.5 \
+    --hash=sha256:680af46846662bb38c5504b78bad9ed9e4f3ba2d54f54ba42494fdf94337fe30 \
+    --hash=sha256:f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
 webencodings==0.5.1 \
     --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
     --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923
@@ -171,3 +171,13 @@ wheel==0.33.6 \
     --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
 wrapt==1.11.2 \
     --hash=sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==19.2.3 \
+    --hash=sha256:340a0ba40fdeb16413914c0fcd8e0b4ebb0bf39a900ec80e11c05d836c05103f \
+    --hash=sha256:e7a31f147974362e6c82d84b91c7f2bdf57e4d3163d3d454e6c3e71944d67135 \
+    # via pipdeptree
+setuptools==41.2.0 \
+    --hash=sha256:4380abcf2a4ffd1a5ba22d687c6d690dce83b2b51c70e9c6d09f7e8c7e8040dc \
+    --hash=sha256:66b86bbae7cc7ac2e867f52dc08a6bd064d938bac59dfec71b9b565dd36d6012 \
+    # via twine

requirements/local.txt

@@ -10,7 +10,7 @@ aspy.yaml==1.3.0          # via pre-commit
 astroid==1.6.6            # via pylint, requirements-detector
 bleach==3.1.0             # via readme-renderer
 bump2version==0.5.11
-certifi==2019.6.16        # via requests
+certifi==2019.9.11        # via requests
 cfgv==2.0.1               # via pre-commit
 chardet==3.0.4            # via requests
 collective.checkdocs==0.2
@@ -44,13 +44,17 @@ requests-toolbelt==0.9.1  # via twine
 requests==2.22.0          # via requests-toolbelt, twine
 requirements-detector==0.6  # via prospector
 setoptconf==0.2.0         # via prospector
-snowballstemmer==1.9.0    # via pydocstyle
+snowballstemmer==1.9.1    # via pydocstyle
 toml==0.10.0              # via pre-commit, tox
-tox==3.13.2
+tox==3.14.0
 tqdm==4.35.0              # via twine
-twine==1.13.0
+twine==1.14.0
 urllib3==1.25.3           # via requests
-virtualenv==16.7.4        # via pre-commit, tox
+virtualenv==16.7.5        # via pre-commit, tox
 webencodings==0.5.1       # via bleach
 wheel==0.33.6
 wrapt==1.11.2             # via astroid
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip==19.2.3               # via pipdeptree
+# setuptools==41.2.0        # via twine

requirements/local27.hash

@@ -1,4 +1,4 @@
-# SHA1:8a359fe6f4198a98f4bc7faba64118b3bf6262b1
+# SHA1:6a89e9082c9a165b7f3ebd30838e19d185f64d38
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -16,9 +16,9 @@ bleach==3.1.0 \
 bump2version==0.5.11 \
     --hash=sha256:524bde030318fe2543038defe0f77739605636fef96924883813cb290cf79c1e \
     --hash=sha256:bfcc051498dda9fd9ac8634689f4516e1c20fdeeace3278932cc6e1248418b36
-certifi==2019.6.16 \
-    --hash=sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939 \
-    --hash=sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695
+certifi==2019.9.11 \
+    --hash=sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50 \
+    --hash=sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef
 chardet==3.0.4 \
     --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
     --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
@@ -123,26 +123,26 @@ requirements-detector==0.6 \
     --hash=sha256:9fbc4b24e8b7c3663aff32e3eba34596848c6b91bd425079b386973bd8d08931
 setoptconf==0.2.0 \
     --hash=sha256:5b0b5d8e0077713f5d5152d4f63be6f048d9a1bb66be15d089a11c898c3cf49c
-snowballstemmer==1.9.0 \
-    --hash=sha256:9f3b9ffe0809d174f7047e121431acf99c89a7040f0ca84f94ba53a498e6d0c9
+snowballstemmer==1.9.1 \
+    --hash=sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e
 toml==0.10.0 \
     --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
     --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e
-tox==3.13.2 \
-    --hash=sha256:dab0b0160dd187b654fc33d690ee1d7bf328bd5b8dc6ef3bb3cc468969c659ba \
-    --hash=sha256:ee35ffce74933a6c6ac10c9a0182e41763140a5a5070e21b114feca56eaccdcd
+tox==3.14.0 \
+    --hash=sha256:0bc216b6a2e6afe764476b4a07edf2c1dab99ed82bb146a1130b2e828f5bff5e \
+    --hash=sha256:c4f6b319c20ba4913dbfe71ebfd14ff95d1853c4231493608182f66e566ecfe1
 tqdm==4.35.0 \
     --hash=sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9 \
     --hash=sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457
-twine==1.13.0 \
-    --hash=sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446 \
-    --hash=sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc
+twine==1.14.0 \
+    --hash=sha256:b2cec0dc1ac55bd74280d257f43763cf0cf928bdcd0de0fd70be70aa1195e3b0 \
+    --hash=sha256:e37d5a73d77b095b85314dde807bfb85b580b5b9d137f5b21332f4636990d97a
 urllib3==1.25.3 \
     --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
     --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232
-virtualenv==16.7.4 \
-    --hash=sha256:94a6898293d07f84a98add34c4df900f8ec64a570292279f6d91c781d37fd305 \
-    --hash=sha256:f6fc312c031f2d2344f885de114f1cb029dfcffd26aa6e57d2ee2296935c4e7d
+virtualenv==16.7.5 \
+    --hash=sha256:680af46846662bb38c5504b78bad9ed9e4f3ba2d54f54ba42494fdf94337fe30 \
+    --hash=sha256:f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
 webencodings==0.5.1 \
     --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
     --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923
@@ -151,3 +151,9 @@ wheel==0.33.6 \
     --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
 wrapt==1.11.2 \
     --hash=sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==41.2.0 \
+    --hash=sha256:4380abcf2a4ffd1a5ba22d687c6d690dce83b2b51c70e9c6d09f7e8c7e8040dc \
+    --hash=sha256:66b86bbae7cc7ac2e867f52dc08a6bd064d938bac59dfec71b9b565dd36d6012 \
+    # via twine

requirements/local27.txt

@@ -10,7 +10,7 @@
 astroid==1.6.6
 bleach==3.1.0             # via readme-renderer
 bump2version==0.5.11
-certifi==2019.6.16        # via requests
+certifi==2019.9.11        # via requests
 chardet==3.0.4            # via requests
 collective.checkdocs==0.2
 docutils==0.15.2          # via collective.checkdocs, readme-renderer
@@ -38,13 +38,16 @@ requests-toolbelt==0.9.1  # via twine
 requests==2.22.0          # via requests-toolbelt, twine
 requirements-detector==0.6  # via prospector
 setoptconf==0.2.0         # via prospector
-snowballstemmer==1.9.0    # via pydocstyle
+snowballstemmer==1.9.1    # via pydocstyle
 toml==0.10.0              # via tox
-tox==3.13.2
+tox==3.14.0
 tqdm==4.35.0              # via twine
-twine==1.13.0
+twine==1.14.0
 urllib3==1.25.3           # via requests
-virtualenv==16.7.4        # via tox
+virtualenv==16.7.5        # via tox
 webencodings==0.5.1       # via bleach
 wheel==0.33.6
 wrapt==1.11.2             # via astroid
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools==41.2.0        # via twine

requirements/py27.hash

@@ -1,4 +1,4 @@
-# SHA1:650a568c6107189c8d5727e5f6808cb5d2ab35ad
+# SHA1:f503ef5dcd0360f47366a72719538b167e746eaa
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -8,9 +8,9 @@
 backports.functools-lru-cache==1.5 \
     --hash=sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a \
     --hash=sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd
-configparser==3.8.1 \
-    --hash=sha256:45d1272aad6cfd7a8a06cf5c73f2ceb6a190f6acc1fa707e7f82a4c053b28b18 \
-    --hash=sha256:bc37850f0cc42a1725a796ef7d92690651bf1af37d744cc63161dac62cabee17
+configparser==4.0.2 \
+    --hash=sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c \
+    --hash=sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df
 enum34==1.1.6 \
     --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
     --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \

requirements/py27.txt

@@ -6,7 +6,7 @@
 #    pip-compile-multi
 #
 backports.functools-lru-cache==1.5
-configparser==3.8.1
+configparser==4.0.2
 enum34==1.1.6
 funcsigs==1.0.2
 future==0.17.1

requirements/test.hash

@@ -1,4 +1,4 @@
-# SHA1:bcb3288f4ef1140f2663a972f527a93f4a9c13a2
+# SHA1:264971e8f9e4a1930809812537ea353b0bd298c2
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -45,9 +45,9 @@ coverage==4.5.4 \
     --hash=sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7 \
     --hash=sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0 \
     --hash=sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025
-importlib-metadata==0.19 \
-    --hash=sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8 \
-    --hash=sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3
+importlib-metadata==0.22 \
+    --hash=sha256:652234b6ab8f2506ae58e528b6fbcc668831d3cc758e1bc01ef438d328b68cdb \
+    --hash=sha256:6f264986fb88042bc1f0535fa9a557e6a376cfe5679dc77caac7fe8b5d43d05f
 mock==3.0.5 \
     --hash=sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3 \
     --hash=sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8
@@ -58,9 +58,9 @@ more-itertools==5.0.0 \
 packaging==19.1 \
     --hash=sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9 \
     --hash=sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe
-pluggy==0.12.0 \
-    --hash=sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc \
-    --hash=sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c
+pluggy==0.13.0 \
+    --hash=sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6 \
+    --hash=sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34
 py==1.8.0 \
     --hash=sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa \
     --hash=sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53

requirements/test.txt

@@ -9,11 +9,11 @@
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via packaging, pytest
 coverage==4.5.4           # via pytest-cov
-importlib-metadata==0.19  # via pluggy, pytest
+importlib-metadata==0.22  # via pluggy, pytest
 mock==3.0.5
 more-itertools==5.0.0
 packaging==19.1           # via pytest
-pluggy==0.12.0            # via pytest
+pluggy==0.13.0            # via pytest
 py==1.8.0                 # via pytest
 pyparsing==2.4.2          # via packaging
 pytest-cov==2.7.1

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{27,35,36,37,py}-{linux,windows}, lint2, lint3, conflicts, checkdocs, verify
+envlist = py{27,35,36,37,py}-{linux,windows}, lint2, lint3, checkdocs, verify
 skip_missing_interpreters = true
 
 [testenv]
@@ -50,14 +50,14 @@ basepython = python2.7
 deps = pip-compile-multi
 commands =
     pip-compile-multi -n local27
-    pip-compile-multi -n local27 -g local27 -i txt -o hash
+    pip-compile-multi -n local27 -g local27 -i txt -o hash --allow-unsafe
 
 [testenv:upgrade3]
 basepython = python3.6
 deps = pip-compile-multi
 commands =
     pip-compile-multi -n local -n testwin
-    pip-compile-multi -n local -n testwin -g local -g testwin -i txt -o hash
+    pip-compile-multi -n local -n testwin -g local -g testwin -i txt -o hash --allow-unsafe
 
 [pytest]
 addopts = -vvvs --doctest-modules