Permalink
Browse files

We can now cleanup things in the virtualenv before package.

We can now force to skip some eggs when creating the package.
Some more docs.
Raised version.
  • Loading branch information...
1 parent 2fd7a7e commit 2cd214725fbd2f2308b56800d89d60815c27dc70 Alvaro Saurin committed Mar 26, 2013
Showing with 114 additions and 68 deletions.
  1. +37 −21 README.md
  2. +55 −38 as/recipe/frozenpkg/frozen.py
  3. +16 −7 as/recipe/frozenpkg/frozenrpm.py
  4. +1 −1 setup.py
  5. +5 −1 testing/buildout.cfg
View
@@ -49,45 +49,61 @@ Options
eggs
The list of eggs that must be copied to the RPM package.
+ eggs-skip
+ A list of eggs to always skip when copying to the package.
+
scripts
The scripts that will be copied to the package. Tese scripts will have their paths relocated to the installation prefix.
extra-dirs
- Any additional directories to create.
+ Any additional directories to create in the package (ie, _"logs"_).
extra-copies
Any additional extra copies. They must be specified as "orig -> dest", where orig can be any valid glob expression, and dest must be a path relative to install-prefix.
+ extra-cleanups
+ Any additional files that must be removed in the package.
+
Example
=======
+
[rpm]
- recipe = as.recipe.frozenpkg:rpm
- pkg-name = testapp
- pkg-version = 1.0
- pkg-vendor = The Vendor
- pkg-packager = My Company
- pkg-url = http://www.mycomp.com
- pkg-license = GPL
- pkg-deps = libevent
- pkg-prefix = /opt/testapp
- pkg-pre-install =
- echo "Installing at ${buildout:pkg-prefix}"
+ recipe = as.recipe.frozenpkg:rpm
+ pkg-name = testapp
+ pkg-version = 1.0
+ pkg-vendor = The Vendor
+ pkg-packager = My Company
+ pkg-url = http://www.mycomp.com
+ pkg-license = GPL
+ pkg-deps =
+ libevent
+ openssl
+ pkg-prefix = /opt/testapp
+ pkg-pre-install =
+ echo "Installing at ${buildout:pkg-prefix}"
pkg-post-install =
- echo "Installed at ${buildout:pkg-prefix}"
+ echo "Installed at ${buildout:pkg-prefix}"
+
+ eggs = ${main:eggs}
+
+ eggs-skip =
+ pip
+ extra-copies =
+ /usr/local/lib/mylib.so -> lib/
+ /usr/local/lib/myextras*.so -> lib/
+ conf/some-local-config.cfg -> conf/
+ extra-dirs =
+ logs
+ var/run
+ extra-cleanups =
+ bin/activate.*
- eggs = ${main:eggs}
-
- extra-copies =
- /usr/local/lib/mylib.so -> lib/
- /usr/local/lib/myextras*.so -> lib/
+ debug = yes
- extra-dirs =
- logs
- debug = yes
@@ -22,6 +22,10 @@
]
+
+def _lst_from_cfg(opt):
+ return [r.strip() for r in opt.split('\n') if r.strip()]
+
################################################################################
class Frozen(object):
@@ -118,10 +122,7 @@ def _copy_eggs (self):
logger.critical('could NOT find easy_install at %s' % easy_install )
sys.exit(1)
- distributions = [
- r.strip()
- for r in self.options.get('eggs', self.name).split('\n')
- if r.strip()]
+ distributions = _lst_from_cfg(self.options.get('eggs', self.name))
import zc.buildout.easy_install
ws = zc.buildout.easy_install.working_set(
@@ -131,38 +132,38 @@ def _copy_eggs (self):
logger.info('Installing eggs in virtualenv.')
for dist in ws:
- if any([re.match(pattern, dist.location) for pattern in SKIP_EGGS]):
- ## skip the eggs in SKIP_EGGS
- logger.debug('... skipping "%s"' % dist.location)
- else:
- logger.debug('... installing "%s" from "%s"' % (dist.key, dist.location))
- args = ['--no-deps']
- try:
- find_links = [
- r.strip()
- for r in self.buildout['buildout']['find-links'].split('\n')
- if r.strip()]
- for l in find_links:
- args += ['--find-links', l]
- except KeyError:
- logger.debug('... no addition find-links')
-
- command = [easy_install] + args + [dist.location]
+ ## check if we must skip this egg
+ skip_eggs = _lst_from_cfg(self.options.get('eggs-skip', '')) + list(SKIP_EGGS)
+ if dist.key in skip_eggs:
+ logger.debug('... skipping "%s"' % dist.key)
+ continue
+
+ logger.info('... installing "%s" from "%s"' % (dist.key, dist.location))
+
+ args = ['--no-deps']
+ try:
+ find_links = _lst_from_cfg(self.buildout['buildout']['find-links'])
+ for l in find_links:
+ args += ['--find-links', l]
+ except KeyError:
+ logger.debug('... no additional find-links')
+
+ command = [easy_install] + args + [dist.location]
+ job = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
+ stdout, _ = job.communicate()
+
+ if job.returncode != 0:
+ logger.debug('...... retrying with easy_install')
+ command = [easy_install] + args + ["%s==%s" % (dist.key, dist.version)]
job = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
stdout, _ = job.communicate()
if job.returncode != 0:
- logger.debug('...... retrying with easy_install')
- command = [easy_install] + args + ["%s==%s" % (dist.key, dist.version)]
- job = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
- stdout, _ = job.communicate()
-
- if job.returncode != 0:
- from zc.buildout import UserError
- msg = 'could NOT run easy_install: %s: %s' % (' '.join(command), stdout)
- logger.critical()
- raise UserError(stdout)
+ from zc.buildout import UserError
+ msg = 'could NOT run easy_install: %s: %s' % (' '.join(command), stdout)
+ logger.critical()
+ raise UserError(stdout)
def _copy_outputs(self):
@@ -198,11 +199,7 @@ def _copy_extra_files (self):
buildout_dir = self.buildout['buildout']['directory']
- extra_copies = [
- r.strip()
- for r in self.options.get('extra-copies', self.name).split('\n')
- if r.strip()]
-
+ extra_copies = _lst_from_cfg(self.options.get('extra-copies', self.name))
logger.info('Copying extras.')
for copy_line in extra_copies:
try:
@@ -238,12 +235,12 @@ def _create_extra_dirs (self):
Create any extra dirs
:param self.virtualenv_dir: the root directory where to copy things
"""
- assert (self.virtualenv_dir != None and len(self.virtualenv_dir) > 0)
+ assert self.virtualenv_dir != None and len(self.virtualenv_dir) > 0
extra_dirs_str = self.options.get('extra-dirs', None)
if extra_dirs_str:
logger.info('Creating extras directories.')
- for directory in [r.strip() for r in extra_dirs_str.split('\n') if r.strip()]:
+ for directory in _lst_from_cfg(extra_dirs_str):
full_path_dest = self._virtualenv_path(directory)
if not os.path.exists(full_path_dest):
@@ -261,6 +258,25 @@ def _create_extra_dirs (self):
raise Exception(msg)
+ def _extra_cleanups(self):
+ """
+ Some extra cleanups in the virtualenv...
+ """
+ extra_cleanups = self.options.get('extra-cleanups', None)
+ if extra_cleanups:
+ logger.debug('Performing extra cleanups.')
+ for cleanup_pattern in _lst_from_cfg(extra_cleanups):
+ for cleanup_file in glob.glob(self._virtualenv_path(cleanup_pattern)):
+ logger.debug('... removing "%s".' % cleanup_file)
+ if os.path.isdir(cleanup_file):
+ shutil.rmtree(cleanup_file, ignore_errors = True)
+ else:
+ try:
+ os.remove(cleanup_file)
+ except IOError, e:
+ logger.error('ERROR: could not remove "%s": %s' % (cleanup_file, str(e)))
+
+
def _prepare_venv(self):
"""
Finish the virtualenv by making it relocatable and removing some extra things we do not need...
@@ -288,6 +304,7 @@ def _prepare_venv(self):
shutil.rmtree(local_dir)
+
def _create_tar (self, filename, compress = False):
"""
Create a tar file from the virtualenv
@@ -60,11 +60,19 @@
%defattr(-, nobody, nobody, 0755)
-@INSTALL_PREFIX@
+@PKG_PREFIX@
# empty lines
"""
+#: directories needed in the rpm build top directory
+RPM_BUILD_DIRS = [
+ "BUILDROOT",
+ "RPMS",
+ "SOURCES",
+ "SPECS",
+ "SRPMS"
+]
###############################################################################
@@ -79,10 +87,6 @@ def install (self):
self._create_rpm_dirs()
- additional_ops = []
- if self.options.has_key('pkg-deps'):
- additional_ops = additional_ops + ["Requires: " + self.options['pkg-deps']]
-
# replace the variables in the "spec" template
rpmspec = RPM_SPEC_TEMPLATE
rpmspec = rpmspec.replace("@TOP_DIR@", self.rpmbuild_dir)
@@ -94,8 +98,12 @@ def install (self):
rpmspec = rpmspec.replace("@PKG_LICENSE@", self.pkg_license)
rpmspec = rpmspec.replace("@PKG_GROUP@", self.pkg_group)
rpmspec = rpmspec.replace("@PKG_AUTODEPS@", self.pkg_autodeps)
- rpmspec = rpmspec.replace("@INSTALL_PREFIX@", self.pkg_prefix)
+ rpmspec = rpmspec.replace("@PKG_PREFIX@", self.pkg_prefix)
rpmspec = rpmspec.replace("@BUILD_ROOT@", self.buildroot)
+
+ additional_ops = []
+ if self.options.has_key('pkg-deps'):
+ additional_ops = additional_ops + ["Requires: " + self.options['pkg-deps']]
rpmspec = rpmspec.replace("@ADDITIONAL_OPS@", "\n".join(additional_ops))
# determine if we must run any pre/post commands
@@ -128,6 +136,7 @@ def install (self):
self._copy_outputs()
self._create_extra_dirs()
self._copy_extra_files()
+ self._extra_cleanups()
self._prepare_venv()
# create a tar file at SOURCES/.
@@ -179,7 +188,7 @@ def _create_rpm_dirs(self):
"""
Create all the top dirs
"""
- for p in ["BUILDROOT", "RPMS", "SOURCES", "SPECS", "SRPMS"]:
+ for p in RPM_BUILD_DIRS:
full_p = os.path.join(self.rpmbuild_dir, p)
if not os.path.exists(full_p):
try:
View
@@ -4,7 +4,7 @@
import os
-__VERSION__ = '0.2.19'
+__VERSION__ = '0.2.20'
@@ -7,7 +7,6 @@ eggs = sh
setproctitle
-
[eggs]
recipe = zc.recipe.egg
eggs = ${buildout:eggs}
@@ -29,9 +28,14 @@ debug = yes
eggs = ${buildout:eggs}
frozen_test_app
+eggs-skip =
+ setproctitle
extra-copies =
buildout.cfg -> extra/data1/buildout.cfg
buildout.cfg -> extra/data2/
extra-dirs =
extra/data1
extra/data3
+extra-cleanups =
+ bin/activate.*
+

0 comments on commit 2cd2147

Please sign in to comment.