Permalink
Browse files

added script code and setup

  • Loading branch information...
1 parent d3db982 commit 261be7520f4b2b4917665eef76828d991eb65bd1 @flupke flupke committed Jan 23, 2013
Showing with 297 additions and 2 deletions.
  1. +11 −0 .gitignore
  2. +2 −0 MANIFEST.in
  3. +23 −0 NEWS.txt
  4. +0 −2 README.md
  5. +49 −0 README.rst
  6. +167 −0 freezerequirements/__init__.py
  7. +45 −0 setup.py
View
@@ -0,0 +1,11 @@
+*.pyc
+
+.installed.cfg
+bin
+develop-eggs
+
+*.egg-info
+
+tmp
+build
+dist
View
@@ -0,0 +1,2 @@
+include README.rst
+include NEWS.txt
View
@@ -0,0 +1,23 @@
+.. This is your project NEWS file which will contain the release notes.
+.. Example: http://www.python.org/download/releases/2.6/NEWS.txt
+.. The content of this file, along with README.rst, will appear in your
+.. project's PyPI page.
+
+News
+====
+
+0.2a1
+-----
+
+*Release date: UNRELEASED*
+
+* Example news entry for the in-development version
+
+
+0.1
+---
+
+*Release date: 15-Mar-2010*
+
+* Example news entry for a released version
+
View
@@ -1,2 +0,0 @@
-freeze-requirements
-===================
View
@@ -0,0 +1,49 @@
+freeze-requirements
+===================
+
+A script to help creating and maintaining frozen requirements for pip.
+
+It is inspired by this `Mozilla dev team blog post <http://blog.mozilla.org/webdev/2013/01/11/switching-to-pip-for-python-deployments/>`_,
+who recently switched to pip for deployment.
+
+Basically it downloads packages from one or more pip 'normal' requirements
+files (the ones you use for development, containing only the 'top level'
+dependencies), and outputs the corresponding list of requirements to copy/paste
+in your frozen production requirements files.
+
+It can also upload the packages to your private pypi repository, and even
+download the packages from there to save bandwidth.
+
+Installation
+------------
+
+Install from pypi::
+
+ $ sudo pip install freeze-requirements
+
+Or from source::
+
+ $ sudo ./setup.py install
+
+Examples
+--------
+
+Download packages locally::
+
+ freeze-requirements requirements.txt --output /tmp/packages
+
+Process multiple requirements files at once::
+
+ freeze-requirements requirements.txt requirements2.txt --output /tmp/packages
+
+Download packages and upload them to a remote host::
+
+ freeze-requirements requirements.txt --upload user@private-pypi.example.com:/home/pypi/packages
+
+Same as above but download packages from the remote host. This may be faster as
+there is no need to upload the packages from your machine and the remote host
+may have a faster internet connection (pip needs to be installed on the remote
+host)::
+
+ freeze-requirements requirements.txt --upload user@private-pypi.example.com:/home/pypi/packages --remote-pip
+
@@ -0,0 +1,167 @@
+import os
+import sys
+import atexit
+import os.path as op
+import subprocess
+import argparse
+import tempfile
+import shutil
+import functools
+import uuid
+
+from setuptools.package_index import distros_for_filename
+from fabric.api import env, run, put
+from fabric.contrib.files import exists
+
+
+TEMPFILES_PREFIX = 'freeze-requirements-'
+
+
+def main():
+ # Parse arguments
+ parser = argparse.ArgumentParser(description='Download dependencies '
+ 'from requirements file(s) and upload them to your private pypi '
+ 'repository')
+ parser.add_argument('requirements', nargs='+',
+ help='a pip requirements file, you can specify multiple '
+ 'requirements files if needed')
+ parser.add_argument('--output', '-o', help='put downloaded files here')
+ parser.add_argument('--remote-pip', '-r', action='store_true',
+ help='run pip on the destination host')
+ parser.add_argument('--upload', '-u', help='upload files here; use '
+ 'user@host:/remote/dir syntax')
+ options = parser.parse_args()
+
+ # Verify options
+ if not options.output and not options.upload:
+ print 'You must specify either --upload or --output'
+ sys.exit(1)
+ if options.output and options.remote_pip:
+ print "You can't use --output with --remote-pip"
+ sys.exit(1)
+ if options.output:
+ if not op.isdir(options.output):
+ print 'Output directory does not exist: %s' % options.output
+ sys.exit(1)
+ output_dir = options.output
+ else:
+ output_dir = tempfile.mkdtemp(prefix=TEMPFILES_PREFIX)
+ atexit.register(shutil.rmtree, output_dir)
+
+ if options.remote_pip and not options.upload:
+ print 'You must specify --upload to use --remote-pip'
+ sys.exit(1)
+
+ if options.upload:
+ try:
+ env.host_string, remote_dir = options.upload.split(':', 1)
+ except ValueError:
+ print 'Invalid upload destination: %s' % options.upload
+ sys.exit(1)
+
+ original_requirements = options.requirements
+
+ # Alias functions to run pip locally or on the remote host
+ if options.remote_pip:
+ run_pip = run
+ mkdtemp = remote_mkdtemp
+ listdir = remote_listdir
+ rmtree = remote_rmtree
+ put_package = remote_move
+ move = remote_move
+ # Upload requirements files to a temp directory
+ print '-' * 78
+ print 'Uploading requirements...'
+ temp_dir = remote_mkdtemp(prefix=TEMPFILES_PREFIX)
+ atexit.register(run, 'rm -rf %s' % temp_dir)
+ remote_requirements = []
+ for i, requirement in enumerate(options.requirements):
+ req_dir = op.join(temp_dir, str(i))
+ run('mkdir %s' % req_dir)
+ remote_requirements.extend(put(requirement, req_dir))
+ options.requirements = remote_requirements
+ output_dir = op.join(temp_dir, 'packages')
+ run('mkdir %s' % output_dir)
+ print
+ else:
+ run_pip = functools.partial(subprocess.check_call, shell=True)
+ mkdtemp = tempfile.mkdtemp
+ listdir = os.listdir
+ rmtree = shutil.rmtree
+ put_package = put
+ move = local_move
+
+ # Download packages
+ print '-' * 78
+ print 'Downloading packages...'
+ requirements_packages = []
+ for original_requirement, requirement in zip(
+ original_requirements, options.requirements):
+ temp_dir = mkdtemp(prefix=TEMPFILES_PREFIX)
+ atexit.register(rmtree, temp_dir)
+ run_pip('pip install -r %s --download %s' % (requirement, temp_dir))
+ requirements_packages.append((original_requirement, listdir(temp_dir)))
+ move(op.join(temp_dir, '*'), output_dir)
+ print
+
+ # Upload or move packages to their final destination
+ if options.upload:
+ print '-' * 78
+ if options.remote_pip:
+ print 'Moving packages to their final destination...'
+ else:
+ print 'Uploading packages...'
+ packages = [op.join(output_dir, p) for p in listdir(output_dir)]
+ created_dirs = set()
+ for package in packages:
+ distro = list(distros_for_filename(package))[0]
+ dst_dir = op.join(remote_dir, distro.key)
+ if dst_dir not in created_dirs:
+ run('mkdir -p %s' % dst_dir)
+ created_dirs.add(dst_dir)
+ put_package(package, dst_dir)
+ print
+
+ # Print frozen requirements for each input requirements file
+ print '-' * 78
+ for requirements_file, packages in requirements_packages:
+ print 'Frozen requirements for "%s":' % requirements_file
+ print
+ for package in packages:
+ distro = list(distros_for_filename(package))[0]
+ print '%s==%s' % (distro.key, distro.version)
+ print
+
+
+def remote_move(src, dst):
+ """
+ Move a file on a remote host.
+ """
+ run('mv -fv %s %s' % (src, dst))
+
+
+def local_move(src, dst):
+ """
+ Move a file on local host.
+ """
+ subprocess.call('mv -fv %s %s' % (src, dst), shell=True)
+
+
+def remote_mkdtemp(prefix='', dir='/tmp'):
+ """
+ Create a remote temporary directory.
+ """
+ while True:
+ temp_dir = op.join(dir, '%s%s' % (prefix, uuid.uuid4().hex))
+ if not exists(temp_dir):
+ run('mkdir %s' % temp_dir)
+ break
+ return temp_dir
+
+
+def remote_listdir(location):
+ return run('ls %s' % location).split()
+
+
+def remote_rmtree(location):
+ run('rm -rf %s' % location)
View
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+from setuptools import setup, find_packages
+import os
+
+here = os.path.abspath(os.path.dirname(__file__))
+README = open(os.path.join(here, 'README.rst')).read()
+NEWS = open(os.path.join(here, 'NEWS.txt')).read()
+
+
+version = '0.1'
+
+install_requires = [
+ # List your project dependencies here.
+ # For more details, see:
+ # http://packages.python.org/distribute/setuptools.html#declaring-dependencies
+]
+
+
+setup(name='freeze-requirements',
+ version=version,
+ description="A script to help creating and maintaining frozen requirements for pip",
+ long_description=README + '\n\n' + NEWS,
+ classifiers=[
+ # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+ 'License :: OSI Approved :: BSD License',
+ 'Operating System :: POSIX',
+ 'Intended Audience :: Developers',
+ 'Topic :: Software Development :: Build Tools',
+ 'Topic :: System :: Software Distribution',
+ ],
+ keywords='pip requirements frozen',
+ author='Luper Rouch',
+ author_email='luper.rouch@gmail.com',
+ url='https://github.com/Stupeflix/freeze-requirements',
+ license='BSD',
+ packages=find_packages(),
+ include_package_data=True,
+ zip_safe=False,
+ install_requires=install_requires,
+ entry_points={
+ 'console_scripts': [
+ 'freeze-requirements=freezerequirements:main'
+ ]
+ }
+)

0 comments on commit 261be75

Please sign in to comment.