add option to pass any additional classpath while launching a topology

heron/tools/cli/src/python/args.py

@@ -28,7 +28,6 @@ def add_titles(parser):
   return parser
 
 
-
 def add_verbose(parser):
   '''
   :param parser:
@@ -143,3 +142,15 @@ def add_deactive_deploy(parser):
       metavar='(a boolean; default: "false")',
       default=False)
   return parser
+
+
+def add_extra_launch_classpath(parser):
+  '''
+  :param parser:
+  :return:
+  '''
+  parser.add_argument(
+      '--extra-launch-classpath',
+      metavar='(a string; additional JVM class path for launching topology)',
+      default="")
+  return parser

heron/tools/cli/src/python/execute.py

@@ -54,7 +54,7 @@ def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=N
   # the java opts must be passed as part of the list
   all_args = [config.get_java_path(), "-client", "-Xmx1g"] + \
              java_opts + \
-             ["-cp", config.get_classpath(lib_jars + extra_jars)]
+             ["-cp", config.get_classpath(extra_jars + lib_jars)]
 
   all_args += [class_name] + list(args)
 

heron/tools/cli/src/python/main.py

@@ -67,7 +67,6 @@ def create_parser():
   Main parser
   :return:
   '''
-  #parser = argparse.ArgumentParser(
   parser = hrc_parse.HeronArgumentParser(
       prog='heron',
       epilog=HELP_EPILOG,

heron/tools/cli/src/python/submit.py

@@ -27,7 +27,9 @@
 import heron.tools.cli.src.python.jars as jars
 import heron.tools.cli.src.python.opts as opts
 import heron.tools.common.src.python.utils.config as config
+import heron.tools.common.src.python.utils.classpath as classpath
 
+# pylint: disable=too-many-return-statements
 
 ################################################################################
 def create_parser(subparsers):
@@ -50,6 +52,7 @@ def create_parser(subparsers):
   cli_args.add_topology_class(parser)
   cli_args.add_config(parser)
   cli_args.add_deactive_deploy(parser)
+  cli_args.add_extra_launch_classpath(parser)
   cli_args.add_system_property(parser)
   cli_args.add_verbose(parser)
 
@@ -100,12 +103,13 @@ def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
   lib_jars = config.get_heron_libs(
       jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
   )
+  extra_jars = cl_args['extra_launch_classpath'].split(':')
 
   # invoke the submitter to submit and launch the topology
   execute.heron_class(
-      'com.twitter.heron.scheduler.SubmitterMain',
-      lib_jars,
-      extra_jars=[],
+      class_name='com.twitter.heron.scheduler.SubmitterMain',
+      lib_jars=lib_jars,
+      extra_jars=extra_jars,
       args=args,
       java_defines=[]
   )
@@ -174,8 +178,8 @@ def submit_fatjar(cl_args, unknown_args, tmp_dir):
   topology_file = cl_args['topology-file-name']
   try:
     execute.heron_class(
-        cl_args['topology-class-name'],
-        config.get_heron_libs(jars.topology_jars()),
+        class_name=cl_args['topology-class-name'],
+        lib_jars=config.get_heron_libs(jars.topology_jars()),
         extra_jars=[topology_file],
         args=tuple(unknown_args),
         java_defines=cl_args['topology_main_jvm_property'])
@@ -294,6 +298,13 @@ def run(command, parser, cl_args, unknown_args):
     Log.error("Unknown file type. Please use .tar or .tar.gz or .jar or .pex file")
     return False
 
+  # check if extra launch classpath is provided and if it is validate
+  if cl_args['extra_launch_classpath']:
+    valid_classpath = classpath.valid_java_classpath(cl_args['extra_launch_classpath'])
+    if not valid_classpath:
+      Log.error("One of jar or directory in extra launch classpath does not exist")
+      return False
+
   # create a temporary directory for topology definition file
   tmp_dir = tempfile.mkdtemp()
 

heron/tools/common/src/python/utils/classpath.py

@@ -0,0 +1,46 @@
+# Copyright 2016 Twitter. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''classpath.py: util functions for verifying a java class path, mainly for heron-cli'''
+
+import os
+
+from heron.common.src.python.utils.log import Log
+
+def valid_path(path):
+  '''
+  Check if an entry in the class path exists as either a directory or a file
+  '''
+  # check if the class path directory exists
+  if path.endswith('*'):
+    Log.debug('Checking class path directory: %s', path[-1])
+    if os.path.isdir(path[-1]):
+      return True
+    return False
+
+  # check if the class path file exists
+  Log.debug('Checking class path file: %s', path)
+  if os.path.isfile(path):
+    return True
+  return False
+
+
+def valid_java_classpath(classpath):
+  '''
+  Given a java classpath, check whether the path entries are valid or not
+  '''
+  paths = classpath.split(':')
+  for path_entry in paths:
+    if not valid_path(path_entry.strip()):
+      return False
+  return True