Enable framework-specific install directories and environment

This commit is my proof-of concept, only the 'go' test works right now. You can look at go/ to see the changes required on a per-framework basis - essentially I've modified go/setup.py to move the declaration of the environment into go/bash_profile.sh Each framework now gets a personal <framework>/installs directory, and can declare any needed environment variables in <framework>/bash_profile.sh, such as modifications to the PATH. The FwBm code now provides an FWROOT environment variable to *all* shell scripts (instead of ~/Frameworks being assumed). Also, an IROOT environment variable is provided to any framework-specific files (e.g. install.sh and bash_profile.sh) for the location of the install directory for that framework. See go/bash_profile.sh to see me using both IROOT and FWROOT. By using IROOT, the strategy for installation is still controlled by python code and the frameworks just reference IROOT and FWROOT in their scripts. These variables are a nice step towards solving TechEmpower#448 The new function replace_environ in setup_util.py goes a long way towards addressing TechEmpower#899 - as you can see I've removed all the changes to the ~/.bash_profile from prerequisites.sh (and this commit serves as proof that everything still works) and most of the changes from config/benchmark_profile
hamiltont · Jul 18, 2014 · 1f10e66 · 1f10e66
1 parent a6b3ab4
commit 1f10e66
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 150 deletions.
diff --git a/go/bash_profile.sh b/go/bash_profile.sh
@@ -1,7 +1,6 @@
-# Set the root of our go installation
-export GOROOT=${IROOT}/go
+# IROOT=${FWROOT}/go/installs
 
-# Where to find the go executable
+export GOROOT=${IROOT}/go
 export PATH="$GOROOT/bin:$PATH"
 
-export GOPATH=${FWROOT}/go
+export GOPATH=${FWROOT}/go
diff --git a/toolset/benchmark/framework_test.py b/toolset/benchmark/framework_test.py
@@ -11,7 +11,6 @@
 import traceback
 import json
 import textwrap
-import logging
 
 class FrameworkTest:
   ##########################################################################################
@@ -29,29 +28,29 @@ class FrameworkTest:
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Primer {name}"
-    echo " {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
+    echo " {wrk} {headers} -d 5 -c 8 -t 8 \"http://{server_host}:{port}{url}\""
     echo "---------------------------------------------------------"
     echo ""
-    {wrk} {headers} -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
+    {wrk} {headers} -d 5 -c 8 -t 8 "http://{server_host}:{port}{url}"
     sleep 5
     
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Warmup {name}"
-    echo " {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}\""
+    echo " {wrk} {headers} -d {duration} -c {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}\""
     echo "---------------------------------------------------------"
     echo ""
-    {wrk} {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}"
+    {wrk} {headers} -d {duration} -c {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}"
     sleep 5
     for c in {interval}
     do
       echo ""
       echo "---------------------------------------------------------"
       echo " Concurrency: $c for {name}"
-      echo " {wrk} {headers} -d {duration} -c $c --timeout $c -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
+      echo " {wrk} {headers} -d {duration} -c $c -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
       echo "---------------------------------------------------------"
       echo ""
-      {wrk} {headers} -d {duration} -c $c --timeout $c -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
+      {wrk} {headers} -d {duration} -c "$c" -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
       sleep 2
     done
   """
@@ -61,29 +60,29 @@ class FrameworkTest:
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Primer {name}"
-    echo " wrk {headers} -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}2\""
+    echo " wrk {headers} -d 5 -c 8 -t 8 \"http://{server_host}:{port}{url}2\""
     echo "---------------------------------------------------------"
     echo ""
-    wrk {headers} -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}2"
+    wrk {headers} -d 5 -c 8 -t 8 "http://{server_host}:{port}{url}2"
     sleep 5
     
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Warmup {name}"
-    echo " wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}2\""
+    echo " wrk {headers} -d {duration} -c {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}2\""
     echo "---------------------------------------------------------"
     echo ""
-    wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}2"
+    wrk {headers} -d {duration} -c {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}2"
     sleep 5
     for c in {interval}
     do
       echo ""
       echo "---------------------------------------------------------"
       echo " Queries: $c for {name}"
-      echo " wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}$c\""
+      echo " wrk {headers} -d {duration} -c {max_concurrency} -t {max_threads} \"http://{server_host}:{port}{url}$c\""
       echo "---------------------------------------------------------"
       echo ""
-      wrk {headers} -d {duration} -c {max_concurrency} --timeout {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}$c"
+      wrk {headers} -d {duration} -c {max_concurrency} -t {max_threads} "http://{server_host}:{port}{url}$c"
       sleep 2
     done
   """
@@ -287,16 +286,9 @@ def validatePlaintext(self, jsonString, out, err):
   # Start the test using it's setup file
   ############################################################
   def start(self, out, err):
-    # Load profile for this installation
     profile="%s/bash_profile.sh" % self.directory
-    if not os.path.exists(profile):
-      logging.warning("Framework %s does not have a bash_profile" % self.name)
-      profile="$FWROOT/config/benchmark_profile"
-
-    set_iroot="export IROOT=%s" % self.install_root
-    setup_util.replace_environ(config=profile, command=set_iroot)
-
-    return self.setup_module.start(self, out, err)
+    setup_util.replace_environ(config=profile, command="IROOT=$FWROOT/go/installs")
+    return self.setup_module.start(self.benchmarker, out, err)
   ############################################################
   # End start
   ############################################################
@@ -926,15 +918,6 @@ def __init__(self, name, directory, benchmarker, runTests, args):
     self.directory = directory
     self.benchmarker = benchmarker
     self.runTests = runTests
-    self.fwroot = benchmarker.fwroot
-
-    # setup logging
-    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
-
-    self.install_root="%s/%s" % (self.fwroot, "installs")
-    if benchmarker.install_strategy is 'pertest':
-      self.install_root="%s/pertest/%s" % (self.install_root, name)
-
     self.__dict__.update(args)
 
     # ensure directory has __init__.py file so that we can use it as a Python package

diff --git a/toolset/run-tests.py b/toolset/run-tests.py
@@ -14,13 +14,6 @@
 # Main
 ###################################################################################################
 def main(argv=None):
-    ''' Runs the program. There are three ways to pass arguments 
-    1) environment variables TFB_*
-    2) configuration file benchmark.cfg
-    3) command line flags
-    In terms of precedence, 3 > 2 > 1, so config file trumps environment variables
-    but command line flags have the final say
-    '''
     # Do argv default this way, as doing it in the functional declaration sets it at compile time
     if argv is None:
         argv = sys.argv
@@ -35,11 +28,7 @@ def main(argv=None):
     sys.path.append('toolset/setup/linux')
 
     # Update environment for shell scripts
-    fwroot = setup_util.get_fwroot()
-    if not fwroot: 
-        fwroot = os.getcwd()
-    setup_util.replace_environ(config='config/benchmark_profile', root=fwroot)
-    print "FWROOT is %s"%setup_util.get_fwroot()
+    setup_util.replace_environ(config='config/benchmark_profile', root=os.getcwd())
 
     conf_parser = argparse.ArgumentParser(
         description=__doc__,
@@ -94,15 +83,12 @@ def main(argv=None):
     parser.add_argument('--database-identity-file', default=dbIdenFile, dest='database_identity_file',
                         help='The key to use for SSH to the database instance.  If not provided, defaults to the value of --client-identity-file.')
     parser.add_argument('-p', dest='password_prompt', action='store_true', help='Prompt for password')
-
+    parser.add_argument('--install-software', action='store_true', help='runs the installation script before running the rest of the commands')
 
     # Install options
     parser.add_argument('--install', choices=['client', 'database', 'server', 'all'], default=None,
                         help='Runs installation script(s) before continuing on to execute the tests.')
     parser.add_argument('--install-error-action', choices=['abort', 'continue'], default='continue', help='action to take in case of error during installation')
-    parser.add_argument('--install-strategy', choices=['unified', 'pertest'], default='pertest', 
-        help='''Affects `--install server`: With unified, all server software is installed into a single directory. 
-        With pertest each test gets its own installs directory, but installation takes longer''')
 
     # Test options
     parser.add_argument('--test', nargs='+', help='names of tests to run')
@@ -150,34 +136,6 @@ def main(argv=None):
 
     benchmarker = Benchmarker(vars(args))
 
-    # Ensure a consistent environment for any subprocesses run during
-    # the lifetime of this program
-    # Note: This will not work if your command starts with 'sudo', you
-    # will need sudo sh -c ". config/benchmark_profile && your_command"
-    setup_env = '. config/benchmark_profile && env'
-    mini_environ = os.environ.copy()
-    mini_environ.clear()
-    mini_environ['HOME']=os.environ['HOME']
-    mini_environ['PATH']=os.environ['PATH']
-    mini_environ['USER']=os.environ['USER']
-    fwroot=subprocess.check_output("pwd", shell=True)
-    mini_environ['FWROOT']=fwroot
-    os.environ.clear()
-    env = subprocess.check_output(setup_env, shell=True, env=mini_environ)
-    for line in env.split('\n'):
-        try:
-            split=line.index('=')
-            key=line[:split]
-            value=line[split+1:]
-            os.environ[key]=value
-        except:
-            print "WARN: Cannot parse %s from config/benchmark_profile" % line
-            continue
-
-    out = subprocess.check_output('env', shell=True)
-    print 'Checking environment'
-    print out
-
     # Run the benchmarker in the specified mode
     if benchmarker.list_tests:
       benchmarker.run_list_tests()

diff --git a/toolset/setup/linux/installer.py b/toolset/setup/linux/installer.py
@@ -6,6 +6,7 @@
 import sys
 import glob
 import logging
+import setup_util
 
 class Installer:
 
@@ -32,8 +33,9 @@ def install_software(self):
   def __install_server_software(self):
     print("\nINSTALL: Installing server software\n")
 
-    bash_functions_path='../toolset/setup/linux/bash_functions.sh'
-    prereq_path='../toolset/setup/linux/prerequisites.sh'
+    # Install global prerequisites
+    bash_functions_path='$FWROOT/toolset/setup/linux/bash_functions.sh'
+    prereq_path='$FWROOT/toolset/setup/linux/prerequisites.sh'
     self.__run_command(". %s && . %s" % (bash_functions_path, prereq_path))
 
     # Pull in benchmarker include and exclude list
@@ -42,25 +44,47 @@ def __install_server_software(self):
     if exclude == None:
         exclude = []
 
-    # Assume we are running from FrameworkBenchmarks
-    install_files = glob.glob('*/install.sh')
+    # Locate all known tests
+    install_files = glob.glob("%s/*/install.sh" % self.root)
 
-    for install_file in install_files:
-        test = os.path.dirname(install_file)
-
-        if test in exclude:
-            logging.debug("%s has been excluded", test)
+    # Run install for selected tests
+    for test_install_file in install_files:
+        test_dir = os.path.dirname(test_install_file)
+        test_name = os.path.basename(test_dir)
+        test_rel_dir = self.__path_relative_to_root(test_dir)
+
+        if test_name in exclude:
+            logging.debug("%s has been excluded", test_name)
             continue
-        elif include is not None and test not in include:
-            logging.debug("%s not in include list", test)
+        elif include is not None and test_name not in include:
+            logging.debug("%s not in include list", test_name)
             continue
         else:
-            logging.debug("Running installer for %s", test)
-            bash_functions_path="../toolset/setup/linux/bash_functions.sh"
-            self.__run_command(". %s && . ../%s" % (bash_functions_path, install_file))
+            logging.info("Running installation for %s"%test_name)
 
-    self.__run_command("sudo apt-get -y autoremove");
-
+            # Find installation directory e.g. FWROOT/go/installs
+            test_install_dir="%s/%s" % (test_dir, self.install_dir)
+            test_rel_install_dir=self.__path_relative_to_root(test_install_dir)
+            if not os.path.exists(test_install_dir):
+              os.makedirs(test_install_dir)
+
+            # Load profile for this installation
+            test_profile_file="%s/bash_profile.sh" % test_dir
+            if os.path.exists(test_profile_file):
+              setup_util.replace_environ(config=test_profile_file)
+            else:
+              logging.warning("Framework %s does not have a bash_profile"%test_name)
+
+            # Find relative installation file
+            test_rel_install_file = "$FWROOT%s"%self.__path_relative_to_root(test_install_file)
+
+            # Then run test installer file
+            # Give all installers FWROOT, IROOT, and bash_functions
+            self.__run_command("IROOT=$FWROOT%s . %s && . %s" % 
+              (test_rel_install_dir, bash_functions_path, test_rel_install_file),
+                cwd=test_install_dir)
+
+    self.__run_command("sudo apt-get -y autoremove");    
 
     print("\nINSTALL: Finished installing server software\n")
   ############################################################
@@ -247,10 +271,8 @@ def __path_exists(self, path, cwd=None):
   # __run_command
   ############################################################
   def __run_command(self, command, send_yes=False, cwd=None, retry=False):
-    try:
-      cwd = os.path.join(self.install_dir, cwd)
-    except AttributeError:
-      cwd = self.install_dir
+    if cwd is None: 
+        cwd = self.install_dir
 
     if retry:
       max_attempts = 5
@@ -261,12 +283,13 @@ def __run_command(self, command, send_yes=False, cwd=None, retry=False):
     if send_yes:
       command = "yes yes | " + command
 
-
-    print("\nINSTALL: %s (cwd=%s)" % (command, cwd))
+    rel_cwd = self.__path_relative_to_root(cwd)
+    print("INSTALL: %s (cwd=%s)" % (command, rel_cwd))
 
     while attempt <= max_attempts:
       error_message = ""
       try:
+
         # Execute command.
         subprocess.check_call(command, shell=True, cwd=cwd, executable='/bin/bash')
         break  # Exit loop if successful.
@@ -303,6 +326,20 @@ def __bash_from_string(self, command):
   # End __bash_from_string
   ############################################################
 
+  ############################################################
+  # __path_relative_to_root
+  # Returns path minus the FWROOT prefix. Useful for clean 
+  # presentation of paths 
+  # e.g. /foo/bar/benchmarks/go/bash_profile.sh
+  # v.s. FWROOT/go/bash_profile.sh 
+  ############################################################
+  def __path_relative_to_root(self, path):
+    # Requires bash shell parameter expansion
+    return subprocess.check_output("D=%s && printf ${D#%s}"%(path, self.root), shell=True, executable='/bin/bash')
+  ############################################################
+  # End __path_relative_to_root
+  ############################################################
+
   ############################################################
   # __download
   # Downloads a file from a URI.
@@ -326,6 +363,7 @@ def __download(self, uri, filename=""):
   def __init__(self, benchmarker):
     self.benchmarker = benchmarker
     self.install_dir = "installs"
+    self.root = subprocess.check_output('printf $FWROOT', shell=True)
 
     # setup logging
     logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)

diff --git a/toolset/setup/linux/prerequisites.sh b/toolset/setup/linux/prerequisites.sh
@@ -35,9 +35,6 @@ fw_exists ~/.bash_profile.bak
   mv ~/.bash_profile ~/.bash_profile.bak
 }
 
-cp ../config/benchmark_profile ~/.bash_profile
-cat ../config/benchmark_profile >> ~/.profile
-cat ../config/benchmark_profile >> ~/.bashrc
 sudo sh -c "echo '*               -    nofile          65535' >> /etc/security/limits.conf"
 
 touch fwbm_prereqs_installed