Skip to content

Commit

Permalink
Merge pull request #2 from boegel/anaconda
Browse files Browse the repository at this point in the history 
rework EB_CondaCreate as generic CondaCreate easyblock, get rid of trivial run_cmd wrappers pre_install_step/post_install_step
  • Loading branch information
@jerowe
jerowe committed Nov 5, 2016
2 parents 38411c9 + 4b29cfc commit 397d27c
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 106 deletions.
30 changes: 5 additions & 25 deletions easybuild/easyblocks/a/anaconda.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,33 +42,11 @@

def set_conda_env(installdir):
""" Set the correct environmental variables for conda """
myEnv = os.environ.copy()
env.setvar('PATH', "{}/bin".format(installdir) + ":" + myEnv["PATH"])
env.setvar('PATH', os.path.join(installdir, 'bin') + ':' + os.environ.get('PATH', ''))
env.setvar('CONDA_ENV', installdir)
env.setvar('CONDA_DEFAULT_ENV', installdir)


def pre_install_step(log, pre_install_cmd = None):
""" User defined pre install step """
if not pre_install_cmd:
pass
else:
log.debug('Pre command run', pre_install_cmd)
run_cmd(pre_install_cmd, log_all=True, simple=True)
log.info('Pre command run {}'.format(pre_install_cmd))


def post_install_step(log, installdir, post_install_cmd):
""" User defined post install step """
if not post_install_cmd:
pass
else:
log.debug('Post command run', post_install_cmd)
set_conda_env(installdir)
run_cmd(post_install_cmd, log_all=True, simple=True)
log.info('Post command run {}'.format(post_install_cmd))


def initialize_conda_env(installdir):
""" Initialize the conda env """
rmtree2(installdir)
Expand All @@ -92,7 +70,8 @@ def extra_options(extra_vars=None):
def install_step(self):
"""Copy all files in build directory to the install directory"""

pre_install_step(self.log, self.cfg['pre_install_cmd'])
if self.cfg['pre_install_cmd']:
run_cmd(self.cfg['pre_install_cmd'], log_all=True, simple=True)

rmtree2(self.installdir)
install_script = self.src[0]['name']
Expand All @@ -103,7 +82,8 @@ def install_step(self):
self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)

post_install_step(self.log, self.installdir, self.cfg['post_install_cmd'])
if self.cfg['post_install_cmd']:
run_cmd(self.cfg['post_install_cmd'], log_all=True, simple=True)

def make_module_req_guess(self):
"""
Expand Down
42 changes: 21 additions & 21 deletions easybuild/easyblocks/c/condaenv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import rmtree2
from easybuild.tools.run import run_cmd
from easybuild.easyblocks.anaconda import post_install_step, pre_install_step, initialize_conda_env, set_conda_env
from easybuild.easyblocks.anaconda import initialize_conda_env, set_conda_env

class EB_CondaEnv(EasyBlock):
"""Support for building/installing environments using conda env. You must use conda-env >= 2.5.2"""
Expand All @@ -48,10 +48,10 @@ def extra_options(extra_vars=None):
"""Extra easyconfig parameters specific to EB_CondaEnv easyblock."""
extra_vars = EasyBlock.extra_options(extra_vars)
extra_vars.update({
'remote_environment': [None, "Remote definition file", CUSTOM],
'environment_file': [None, "Environment.yml file. Either add a 'environment.yml' to your sources, or specify the full path here.", CUSTOM],
'post_install_cmd': [None, "Commands after install: pip install, cpanm install, etc", CUSTOM],
'pre_install_cmd': [None, "Commands before install: setting environment variables, etc", CUSTOM],
'remote_environment': [None, "Remote definition file", CUSTOM],
})
return extra_vars

Expand Down Expand Up @@ -88,10 +88,26 @@ def build_step(self):
def install_step(self):
"""Copy all files in build directory to the install directory"""

pre_install_step(self.log, self.cfg['pre_install_cmd'])
if self.cfg['pre_install_cmd']:
run_cmd(self.cfg['pre_install_cmd'], log_all=True, simple=True)

initialize_conda_env(self.installdir)
self.install_conda_env()
post_install_step(self.log, self.installdir, self.cfg['post_install_cmd'])

set_conda_env(self.installdir)

if self.cfg['environment_file']:
cmd = "conda env create -f {} -p {}".format(self.cfg['environment_file'], self.installdir)
elif self.cfg['remote_environment']:
cmd = "conda env create {} -p {}".format(self.cfg['remote_environment'], self.installdir)
else:
cmd = "conda env create -p {}".format(self.installdir)

run_cmd(cmd, log_all=True, simple=True)

self.log.info('Installed conda env')

if self.cfg['post_install_cmd']:
run_cmd(self.cfg['post_install_cmd'], log_all=True, simple=True)

def make_module_extra(self):
"""Add the install directory to the PATH."""
Expand All @@ -111,19 +127,3 @@ def make_module_req_guess(self):
'MANPATH': ['man', os.path.join('share', 'man')],
'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']],
}

def install_conda_env(self):
""" Install requirements to conda env """

set_conda_env(self.installdir)

if self.cfg['environment_file']:
cmd = "conda env create -f {} -p {}".format(self.cfg['environment_file'], self.installdir)
elif self.cfg['remote_environment']:
cmd = "conda env create {} -p {}".format(self.cfg['remote_environment'], self.installdir)
else:
cmd = "conda env create -p {}".format(self.installdir)

run_cmd(cmd, log_all=True, simple=True)

self.log.info('Installed conda env')
88 changes: 28 additions & 60 deletions easybuild/easyblocks/c/condacreate.py → easybuild/easyblocks/generic/condacreate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,84 +28,66 @@
"""

import os
import shutil
import stat

import easybuild.tools.environment as env
from easybuild.framework.easyblock import EasyBlock
from easybuild.easyblocks.anaconda import initialize_conda_env, set_conda_env
from easybuild.easyblocks.generic.binary import Binary
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import rmtree2
from easybuild.tools.run import run_cmd
from easybuild.easyblocks.anaconda import post_install_step, pre_install_step, initialize_conda_env, set_conda_env

class EB_CondaCreate(EasyBlock):

class CondaCreate(Binary):
"""Support for building/installing CondaCreate."""

@staticmethod
def extra_options(extra_vars=None):
"""Extra easyconfig parameters specific to EB_CondaCreate easyblock."""
extra_vars = EasyBlock.extra_options(extra_vars)
extra_vars = Binary.extra_options(extra_vars)
extra_vars.update({
'requirements': [None, "Requirements files", CUSTOM],
'channels': [None, "Custom conda channels", CUSTOM],
'post_install_cmd': [None, "Commands after install: pip install, cpanm install, etc", CUSTOM],
'pre_install_cmd': [None, "Commands before install: setting environment variables, etc", CUSTOM],
'requirements': [None, "Requirements file", CUSTOM],
})
return extra_vars

def __init__(self, *args, **kwargs):
"""Initialize EB_CondaCreate-specific variables."""
super(EB_CondaCreate, self).__init__(*args, **kwargs)


def extract_step(self):
"""Move all source files to the build directory"""

if not self.src:
pass
"""No sources expected."""
if self.src:
super(EB_CondaCreate, self).extract_step()
else:
self.src[0]['finalpath'] = self.builddir

# copy source to build dir.
for source in self.src:
src = source['path']
dst = os.path.join(self.builddir, source['name'])
try:
shutil.copy2(src, self.builddir)
os.chmod(dst, stat.S_IRWXU)
except (OSError, IOError), err:
raise EasyBuildError("Couldn't copy %s to %s: %s", src, self.builddir, err)

def configure_step(self):
"""No configuration, this is binary software"""
pass

def build_step(self):
"""No compilation, this is binary software"""
pass
pass

def install_step(self):
"""Copy all files in build directory to the install directory"""
"""Set up conda environment using 'conda create' and install using specified requirements."""

if self.cfg['pre_install_cmd']:
run_cmd(self.cfg['pre_install_cmd'], log_all=True, simple=True)

pre_install_step(self.log, self.cfg['pre_install_cmd'])
initialize_conda_env(self.installdir)

cmd = "conda create -y -p {}".format(self.installdir)
cmd = "conda create -y -p %s" % self.installdir
run_cmd(cmd, log_all=True, simple=True)

if self.cfg['requirements']:
self.install_conda_requirements()
set_conda_env(self.installdir)

if self.cfg['channels'] and self.cfg['requirements']:
cmd = "conda install -y -c %s %s" % (self.cfg['channels'], self.cfg['requirements'])
elif self.cfg['requirements']:
cmd = "conda install -y %s" % self.cfg['requirements']

post_install_step(self.log, self.installdir, self.cfg['post_install_cmd'])
run_cmd(cmd, log_all=True, simple=True)
self.log.info('Installed conda requirements')

if self.cfg['post_install_cmd']:
run_cmd(self.cfg['post_install_cmd'], log_all=True, simple=True)

def make_module_extra(self):
"""Add the install directory to the PATH."""

txt = super(EB_CondaCreate, self).make_module_extra()
txt = super(CondaCreate, self).make_module_extra()
txt += self.module_generator.set_environment('CONDA_ENV', self.installdir)
txt += self.module_generator.set_environment('CONDA_DEFAULT_ENV', self.installdir)
self.log.debug("make_module_extra added this: %s" % txt)
self.log.debug("make_module_extra added this: %s", txt)
return txt

def make_module_req_guess(self):
Expand All @@ -117,17 +99,3 @@ def make_module_req_guess(self):
'MANPATH': ['man', os.path.join('share', 'man')],
'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']],
}

def install_conda_requirements(self):
""" Install requirements to conda env """

set_conda_env(self.installdir)

if self.cfg['channels'] and self.cfg['requirements']:
cmd = "conda install -y -c {} {}".format(self.cfg['channels'],
self.cfg['requirements'])
elif self.cfg['requirements']:
cmd = "conda install -y {}".format(self.cfg['requirements'])

run_cmd(cmd, log_all=True, simple=True)
self.log.info('Installed conda requirements')

0 comments on commit 397d27c

Please sign in to comment.