IMPALA-8515: port shell tests to use shell build

shell/make_shell_tarball.sh builds a tarball with all the
shell dependencies bundled. We should test the contents of
that tarball in the shell tests instead of using infra/python/env
and the libraries bundled there.

This tarball is one of the default targets (e.g. run by buildall.sh) so
this should not affect any typical development workflows.

Note that this means the shell tests now requires the shell tarball to
be built locally, which doesn't necessarily happen for remote cluster
tests, so we preserve the old behaviour in that case.

Testing:
Ran core tests on CentOS 6 and CentOS 7.

Change-Id: I581363639b279a9c2ff1fd982bdb140260b24baa
Reviewed-on: http://gerrit.cloudera.org:8080/13267
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

tests/common/environ.py

@@ -29,6 +29,18 @@
 # Default web UI URL for local test cluster
 DEFAULT_LOCAL_WEB_UI_URL = "http://localhost:25000"
 
+# Find the local build version. May be None if Impala wasn't built locally.
+IMPALA_LOCAL_BUILD_VERSION = None
+IMPALA_LOCAL_VERSION_INFO = os.path.join(IMPALA_HOME, "bin/version.info")
+if os.path.isfile(IMPALA_LOCAL_VERSION_INFO):
+  with open(IMPALA_LOCAL_VERSION_INFO) as f:
+    for line in f:
+      match = re.match("VERSION: ([^\s]*)\n", line)
+      if match:
+        IMPALA_LOCAL_BUILD_VERSION = match.group(1)
+  if IMPALA_LOCAL_BUILD_VERSION is None:
+    raise Exception("Could not find VERSION in {0}".format(IMPALA_LOCAL_VERSION_INFO))
+
 # Find the likely BuildType of the running Impala. Assume it's found through the path
 # $IMPALA_HOME/be/build/latest as a fallback.
 build_type_arg_regex = re.compile(r'--build_type=(\w+)', re.I)

tests/shell/test_shell_commandline.py

@@ -30,8 +30,8 @@
 from tests.common.skip import SkipIf
 from tests.common.test_dimensions import create_beeswax_dimension
 from time import sleep, time
-from util import get_impalad_host_port
-from util import assert_var_substitution, run_impala_shell_cmd, ImpalaShell
+from util import (get_impalad_host_port, assert_var_substitution, run_impala_shell_cmd,
+                  ImpalaShell, IMPALA_SHELL_EXECUTABLE)
 from contextlib import closing
 
 DEFAULT_QUERY = 'select 1'
@@ -707,7 +707,7 @@ def test_missing_query_file(self, vector):
   def _validate_expected_socket_connected(self, vector, args, sock):
     # Building an one-off shell command instead of using Util::ImpalaShell since we need
     # to customize the impala daemon socket.
-    shell_cmd = ["{0}/bin/impala-shell.sh".format(os.environ['IMPALA_HOME'])]
+    shell_cmd = [IMPALA_SHELL_EXECUTABLE]
     expected_output = "PingImpalaService"
     with open(os.devnull, 'w') as devnull:
       try:

tests/shell/util.py

@@ -25,8 +25,23 @@
 import time
 from subprocess import Popen, PIPE
 
+from tests.common.environ import (IMPALA_LOCAL_BUILD_VERSION,
+                                  IMPALA_TEST_CLUSTER_PROPERTIES)
 from tests.common.impala_test_suite import IMPALAD_BEESWAX_HOST_PORT
+
+
 SHELL_HISTORY_FILE = os.path.expanduser("~/.impalahistory")
+IMPALA_HOME = os.environ['IMPALA_HOME']
+
+if IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster():
+  # With remote cluster testing, we cannot assume that the shell was built locally.
+  IMPALA_SHELL_EXECUTABLE = os.path.join(IMPALA_HOME, "bin/impala-shell.sh")
+else:
+  # Test the locally built shell distribution.
+  IMPALA_SHELL_EXECUTABLE = os.path.join(
+      IMPALA_HOME, "shell/build", "impala-shell-" + IMPALA_LOCAL_BUILD_VERSION,
+      "impala-shell")
+
 
 def assert_var_substitution(result):
   assert_pattern(r'\bfoo_number=.*$', 'foo_number= 123123', result.stdout, \
@@ -129,8 +144,10 @@ def get_impalad_port(vector):
 def get_shell_cmd(vector):
   """Get the basic shell command to start the shell, given the provided test vector.
   Returns the command as a list of string arguments."""
-  return [os.path.join(os.environ['IMPALA_HOME'], "bin/impala-shell.sh"),
-          "-i{0}".format(get_impalad_host_port(vector))]
+  # Use impala-shell build instead of bin/impala-shell.sh so that we test with the
+  # system python, not the toolchain python and in a configuration close to what
+  # we will distribute.
+  return [IMPALA_SHELL_EXECUTABLE, "-i{0}".format(get_impalad_host_port(vector))]
 
 
 def get_open_sessions_metric(vector):
@@ -206,5 +223,10 @@ def _start_new_shell_process(self, vector, args=None, env=None):
     cmd = get_shell_cmd(vector)
     if args is not None: cmd += args
     if not env: env = os.environ
+    # Don't inherit PYTHONPATH - the shell launch script should set up PYTHONPATH
+    # to include dependencies. Copy 'env' to avoid mutating argument or os.environ.
+    env = dict(env)
+    if "PYTHONPATH" in env:
+      del env["PYTHONPATH"]
     return Popen(cmd, shell=False, stdout=PIPE, stdin=PIPE, stderr=PIPE,
                  env=env)