Merge pull request #39 from keisukefukuda/change-test-script-to-shell

[WIP] Change test script to shell

.travis.yml

@@ -24,6 +24,9 @@ cache:
   - directories:
       - $HOME/mpi
       - $HOME/zsh
+      - $HOME/.mpienv-test
+      - $HOME/.mpienv-build
+      - $HOME/.mpienv-cache
 
 addons:
   apt:
@@ -34,8 +37,8 @@ addons:
       - ibverbs-utils
       - libibverbs-dev
 
-before_script:
-  - sh ./.travis_install.sh
+#before_script:
+#  - sh ./.travis_install.sh
 
 install:
   - pip install -r requirements.txt
@@ -46,5 +49,6 @@ script:
   - autopep8 --diff -r . --global-config .pep8 | tee check_autopep8
   - test ! -s check_autopep8
   - flake8 $(find . -name "*.py")
-  - coverage run $(which nosetests) -v
+  # - coverage run $(which nosetests) -v
+  - ./tests/test_main.sh
   - codeclimate-test-reporter || true # CODECLIMATE_REPO_TOKEN is defined in TravisCI's setting

README.md

@@ -111,8 +111,8 @@ Installed MPIs:
  * openmpi-2.1.1 -> /Users/keisukefukuda/mpi/openmpi-2.1.1
 ```
 
-The '*' mark indicates that the MPI "openmpi-2.1.1" is active, which
-means it's on the `$PATH` and `$LD_LIBRARYPATH` environment variables.
+The `*` mark indicates that the MPI `openmpi-2.1.1` is active, which
+means it's on the `PATH` and `LD_LIBRARYPATH` environment variables.
 You can check that `openmpi-2.1.1` is active.
 
 ```bash
@@ -156,7 +156,7 @@ HYDRA build details:
 "mpich-3.2" is now active. 
 
 ## Running MPI applications
-To run your MPI application, you need to specify a few options to the `mpiexsec` command.
+To run your MPI application, you need to specify a few options to the `mpiexec` command.
 
 ```bash
 $ # If you use Open MPI
@@ -188,7 +188,8 @@ $ mpiexec --genvall -n ${NP} --hostfile ${HOSTFILE} ./your.app
 
 If you use MPI with Python and want to swtich multiple MPI
 installations, what annoys you is that `mpi4py` is tied to a single
-MPI when it is compiled and installed. This means that you have to do
+MPI instance when it is compiled and installed. This means that you
+have to do
 
 ```bash
 $ pip uninstall mpi4py
@@ -224,7 +225,7 @@ $ mpiexec -n 2 python -c "from mpi4py import MPI; print(MPI.COMM_WORLD.Get_rank(
 ```
 
 OK, now your `mpi4py` is properly set up. To run Python script on multiple nodes,
-you need to pass an additional environment variable: `PYTHONPATH`.
+you need to pass an additional environment variable `PYTHONPATH`.
 
 ```bash
 $ # Open MPI
@@ -233,6 +234,3 @@ $ mpiexec --prefix /home/kfukuda/mpi/openmpi-2.1.1 -x PYTHONPATH -n ${NP} --host
 $ # MPICH/MVAPICH
 $ mpiexec --genvall -n ${NP} --hostfile ${HOSTFILE} ./your.app
 ```
-
-
-

bin/build.py

@@ -27,7 +27,7 @@ def main():
     inst = create_installer(manager, args.mpi, args.name,
                             verbose=args.verbose)
 
-    inst.install(npar=args.npar)
+    inst.build(npar=args.npar)
 
 
 if __name__ == "__main__":

bin/configure.py

@@ -27,7 +27,7 @@ def main():
     inst = create_installer(manager, args.mpi, args.name,
                             verbose=args.verbose)
 
-    inst.configure(args.conf_args)
+    inst.configure()
 
 
 if __name__ == "__main__":

bin/install.py

@@ -34,7 +34,7 @@ def main():
     inst = create_installer(manager, args.mpi, args.name,
                             verbose=args.verbose)
 
-    inst.build(npar=args.npar)
+    inst.install(npar=args.npar)
 
 
 if __name__ == "__main__":

common.py

@@ -215,6 +215,7 @@ def __init__(self, root_dir):
         self._root_dir = root_dir
         self._vers_dir = os.path.join(os.environ.get("MPIENV_VERSIONS_DIR") or
                                       os.path.join(root_dir, 'versions'))
+        self._shims_dir = os.path.join(self._vers_dir, 'shims')
         pybin = os.path.realpath(sys.executable)
         pybin_enc = re.sub(r'[^a-zA-Z0-9.]', '_', re.sub('^/', '', pybin))
 
@@ -421,14 +422,13 @@ def use(self, name, mpi4py=False):
                              "'{}'\n".format(name))
             exit(-1)
 
-        shims = os.path.join(self._root_dir, 'shims')
-        if os.path.exists(shims):
-            shutil.rmtree(shims)
+        if os.path.exists(self._shims_dir):
+            shutil.rmtree(self._shims_dir)
 
-        os.mkdir(shims)
+        os.mkdir(self._shims_dir)
 
         for d in ['bin', 'lib', 'include', 'libexec']:
-            dr = os.path.join(shims, d)
+            dr = os.path.join(self._shims_dir, d)
             if not os.path.exists(dr):
                 os.mkdir(dr)
 
@@ -471,8 +471,6 @@ def _mirror_file(self, f, dst_dir):
             os.symlink(src, dst)
 
     def _use_mpich(self, prefix):
-        shims = os.path.join(self._root_dir, 'shims')
-
         bin_files = _glob_list([prefix, 'bin'],
                                ['hydra_*',
                                 'mpi*',
@@ -490,25 +488,22 @@ def _use_mpich(self, prefix):
                                 'primitives'])
 
         for f in bin_files:
-            self._mirror_file(f, os.path.join(shims, 'bin'))
+            self._mirror_file(f, os.path.join(self._shims_dir, 'bin'))
 
         for f in lib_files:
-            self._mirror_file(f, os.path.join(shims, 'lib'))
+            self._mirror_file(f, os.path.join(self._shims_dir, 'lib'))
 
         for f in inc_files:
-            self._mirror_file(f, os.path.join(shims, 'include'))
+            self._mirror_file(f, os.path.join(self._shims_dir, 'include'))
 
     def _use_mvapich(self, prefix):
         self._use_mpich(prefix)
         libexec_files = _glob_list([prefix, 'libexec'],
                                    ['osu-micro-benchmarks'])
-        shims = os.path.join(self._root_dir, 'shims')
         for f in libexec_files:
-            self._mirror_file(f, os.path.join(shims, 'libexec'))
+            self._mirror_file(f, os.path.join(self._shims_dir, 'libexec'))
 
     def _use_openmpi(self, prefix):
-        shims = os.path.join(self._root_dir, 'shims')
-
         bin_files = _glob_list([prefix, 'bin'],
                                ['mpi*',
                                 'ompi-*',
@@ -529,13 +524,13 @@ def _use_openmpi(self, prefix):
                                ['mpi*.h', 'openmpi'])
 
         for f in bin_files:
-            self._mirror_file(f, os.path.join(shims, 'bin'))
+            self._mirror_file(f, os.path.join(self._shims_dir, 'bin'))
 
         for f in lib_files:
-            self._mirror_file(f, os.path.join(shims, 'lib'))
+            self._mirror_file(f, os.path.join(self._shims_dir, 'lib'))
 
         for f in inc_files:
-            self._mirror_file(f, os.path.join(shims, 'include'))
+            self._mirror_file(f, os.path.join(self._shims_dir, 'include'))
 
 
 _root_dir = (os.environ.get("MPIENV_ROOT", None) or

init

@@ -2,15 +2,23 @@
 
 export MPIENV_ROOT="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
 
-if [ ! -f $MPIENV_ROOT/shims ]; then
-    G=$MPIENV_ROOT/version_global
+
+if [ -z "${MPIENV_VERSIONS_DIR}" ]; then
+    export MPIENV_VERSIONS_DIR=$MPIENV_ROOT/versions
+fi
+
+mkdir -p ${MPIENV_VERSIONS_DIR}
+mkdir -p ${MPIENV_VERSIONS_DIR}/shims
+
+if [ ! -f $MPIENV_VERSIONS_DIR/shims ]; then
+    G=$MPIENV_VERSIONS_DIR/version_global
     if [ -f $G ]; then
-        ln -s $(cat $G) $MPIENV_ROOT/shims
+        ln -s $(cat $G) $MPIENV_VERSIONS_DIR/shims
     fi
 fi
 
-export PATH=$MPIENV_ROOT/shims/bin:${PATH:-}
-export LD_LIBRARY_PATH=$MPIENV_ROOT/shims/lib:$MPIENV_ROOT/shims/lib64:${LD_LIBRARY_PATH:-}
+export PATH=$MPIENV_VERSIONS_DIR/shims/bin:${PATH:-}
+export LD_LIBRARY_PATH=$MPIENV_VERSIONS_DIR/shims/lib:$MPIENV_VERSIONS_DIR/shims/lib64:${LD_LIBRARY_PATH:-}
 
 function usage() {
     echo "Usage: mpienv [command] [options...]"
@@ -31,6 +39,8 @@ function mpienv() {
             {
                 eval $(env PYTHONPATH=$MPIENV_ROOT:${PYTHONPATH:-} \
                            python $root/bin/use.py $*)
+                env PYTHONPATH=$MPIENV_ROOT:${PYTHONPATH:-} \
+                           python $root/bin/use.py $*
                 if [ -z "${BASH_VERSION:-}" -a ! -z "${ZSH_VERSION:-}" ]; then
                     rehash
                 fi

mpienv/installer.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 
+import json
 import os
 import os.path
 import re
@@ -93,18 +94,33 @@ def clean(self):
             shutil.rmtree(self.dir_path)
 
     def download(self):
+        # TODO(keisukefukuda): check the checksum
         if not os.path.exists(self.local_file):
             with open(self.local_file, 'w') as f:
                 check_call(['curl', self.url], stdout=f)
 
-    def configure(self, conf_args):
+    def configure(self):
+        # TODO(keisukefukuda): check configure options and
+        #                      re-run ./configure only when necessary
+        # TODO(keisukefukuda): Support multiple verbosity level
+        #                      Level 0: silent
+        #                      Level 1: only prints "Installing..."
+        #                      Level 2: prints everything
         self.download()
+        print('Configuring in {}'.format(self.dir_path))
 
+        print("ext_path={}".format(self.ext_path))
         # Extract the archive files
         if not os.path.exists(self.dir_path):
             check_call(['tar', '-xf', self.local_file],
                        cwd=self.ext_path)
 
+        opts = os.environ.get("MPIENV_CONFIGURE_OPTS")
+        if opts:
+            conf_args = opts.split()
+        else:
+            conf_args = []
+
         # fix the configure argument
         try:
             idx = conf_args.index('--prefix')
@@ -127,29 +143,45 @@ def configure(self, conf_args):
                 conf_args = ['--help']
         except ValueError:
             # if --help is not found
-            conf_args[:-1] = ['--prefix', self.prefix]
+            conf_args += ['--prefix', self.prefix]
 
         print(' '.join(['./configure'] + conf_args))
 
-        # run configure scripts
-        assert(os.path.exists(self.dir_path))
-        check_call(['./configure'] + conf_args,
-                   cwd=self.dir_path)
+        # Check cache
+        cache = os.path.join(self.dir_path, 'mpienv.conf')
+        if os.path.exists(cache):
+            with open(cache, 'r') as f:
+                try:
+                    loaded = json.load(f)
+                    cached = (loaded == conf_args)
+                    print("loaded = {}".format(loaded))
+                    print("conf_args = {}".format(conf_args))
+                except ValueError:
+                    cached = False
+        else:
+            cached = False
+        print("cached = {}".format(cached))
+
+        if cached is False:
+            # run configure scripts
+            assert(os.path.exists(self.dir_path))
+            print(' '.join(['./configure'] + conf_args))
+            check_call(['./configure'] + conf_args,
+                       cwd=self.dir_path)
+            with open(cache, 'w') as f:
+                json.dump(conf_args, f)
 
     def build(self, npar=1):
-        config_log = os.path.join(self.dir_path, 'config.log')
-        if not os.path.exists(config_log):
-            self.configure([])
-
-        # run configure scripts
+        self.configure()
+        print('Building in {}'.format(self.dir_path))
+        # run make
+        print(' '.join(['make', '-j', str(npar)]))
         check_call(['make', '-j', str(npar)],
                    shell=True, cwd=self.dir_path)
 
     def install(self, npar=1):
-        config_log = os.path.join(self.dir_path, 'config.log')
-        if not os.path.exists(config_log):
-            self.configure([])
-
+        self.configure()
+        print(' '.join(['make', 'install', '-j', str(npar)]))
         check_call(['make', 'install', '-j', str(npar)],
                    cwd=self.dir_path)
 
@@ -184,6 +216,9 @@ def create_installer(manager, mpi, name, verbose):
         sys.stderr.write("Error: Unknown MPI: '{}'\n".format(mpi))
         exit(-1)
 
+    if name is None:
+        name = mpi
+
     mpi_type = _list[mpi]['type']
 
     if mpi_type == 'openmpi':

tests/.gitignore

@@ -0,0 +1 @@
+shunit2

tests/test_main.py

@@ -112,36 +112,6 @@ def test_autodiscover_add(self):
                  for ln in lines]
         self.assertEqual(mpi_list, lines)
 
-    def test_list(self):
-        out, err, ret = sh_session([
-            "mpienv autodiscover -q --add ~/mpi",
-            "mpienv list --json"
-        ])
-        self.assertEqual(0, ret)
-
-        data = json.loads(out)
-        self.assertEqual(mpi_list, sorted(data.keys()))
-
-    def test_info(self):
-        for mpi in mpi_list:
-            out, err, ret = sh_session([
-                "mpienv autodiscover -q --add ~/mpi",
-                "mpienv info {} --json".format(mpi)
-            ])
-            self.assertEqual(0, ret)
-
-            data = json.loads(out)
-            self.assertEqual(mpi, data['name'])
-            self.assertEqual(mpi_vers[mpi], data['version'])
-
-            self.assertTrue('broken' in data)
-            self.assertTrue('symlink' in data)
-            self.assertTrue('mpiexec' in data)
-            self.assertTrue('mpicc' in data)
-            self.assertTrue('mpicxx' in data)
-            self.assertTrue('default_name' in data)
-            self.assertTrue('prefix' in data)
-
     def test_list_broken(self):
         out, err, ret = sh_session([
             "mpienv autodiscover -q --add ~/mpi",