added manual mode

Instance.py

@@ -0,0 +1,38 @@
+import boto3
+
+class Instance:
+    def __init__(self, name, ip, port):
+        self.name = name
+        self.ip = ip
+        self.port = port
+
+class RetrieveInstanceIPs():
+    def getIps(self,service,port):
+        raise NotImplementedError("Subclass must implement abstract method")
+
+class ManualInstances(RetrieveInstanceIPs):
+    def getIps(self,service,ips,port):
+        instances=[]
+        for ip in ips.split(','):
+            instances.append(Instance(name=service,ip=ip,port=port))
+        return instances
+
+class AWSInstances(RetrieveInstanceIPs):
+    def __init__(self,profile,region):
+        self.profile = profile
+        self.region = region
+        boto3.setup_default_session(profile_name=profile)
+    def getIps(self,service,port):
+        instances=[]
+        for ec2_ip in self.req_aws_ips(service, self.region):
+            instances.append(Instance(name=service,ip=ec2_ip,port=port))
+        return instances
+    def req_aws_ips(self,service, region):
+        ips=[]
+        aws_filter = lambda name,value: [{'Name':'tag:'+name,'Values':[value]}]
+        client = boto3.client('ec2')
+        response = client.describe_instances(Filters=aws_filter('Name',service))
+        for res in response.get('Reservations'):
+            for instance in res.get('Instances'):
+                ips.append(instance.get(u'PrivateIpAddress'))
+        return ips

README.md

@@ -24,7 +24,17 @@ Install
 Usage
 -----
 
-    $ kafkatunnel ec2-user@awsjumphost
+`kafkatool.py` supports two ways of passing the remote zookeeper/kafka ip's:
+
+* automatic retrival by ec2 resource tags (Name=zookeeper/kafka)
+
+    $ kafkatunnel aws ec2-user@awsjumphost
+
+* manual passing the ip's
+
+    $ kafkatunnel manual 10.11.85.128,10.11.82.30,10.11.83.9 10.11.80.7,10.11.80.123,10.11.81.13
+
+afterwards you have to provide your root password in order to create the interfaces
 
     zookeeper  on 10.11.85.128    port  2181
     zookeeper  on 10.11.82.30     port  2181

kafkatunnel.py

@@ -1,43 +1,49 @@
 #!/usr/bin/env python3
 import click
-import boto3
 import sys
 import subprocess
 
+from Instance import ManualInstances, AWSInstances
 
-@click.command()
+@click.group(help='Access kafka/zookeeper via ssh tunnel to consume and produce messages from your local machine')
+def cli():
+    pass
+
+@cli.command(help='retrieve kafka/zookeeper ip\'s from AWS (important: a resource tag with Name=kafka/zookeeper is needed)')
 @click.argument('jump_host')
 @click.option('-zp','--zookeeper_port',default='2181')
 @click.option('-kp','--kafka_port',default='9091')
 @click.option('-r','--region',default='ap-southeast-2')
 @click.option('-p','--profile',default='default')
-def cli(jump_host, zookeeper_port, kafka_port, region, profile):
+def aws(jump_host,zookeeper_port,kafka_port,region,profile):
     instances=[]
-    click.echo('accessing kafka & zookeeper via ssh ...')
-    boto3.setup_default_session(profile_name=profile)
-    instances += req_instances('zookeeper', zookeeper_port, region)
-    instances += req_instances('kafka', kafka_port, region)
-    print_instances(instances)
-    add_local_interfaces(instances)
-    connect_ssh_tunnel(jump_host,instances)
-    remove_local_interfaces(instances)
+    click.echo('retrieving ip\'s from AWS zookeeper/kafka ec2 instances by tag_name ...')
+    aws = AWSInstances(profile,region)
+    instances += aws.getIps('zookeeper',zookeeper_port)
+    instances += aws.getIps('kafka',kafka_port)
+    connect(jump_host,instances)
+
 
-def req_instances(service, port, region):
+@cli.command(help='provide the IP\'s of your zookeeper/kafka')
+@click.argument('jump_host')
+@click.argument('zookeeper_ips')
+@click.argument('kafka_ips')
+@click.option('-zp','--zookeeper_port',default='2181')
+@click.option('-kp','--kafka_port',default='9091')
+def manual(jump_host,zookeeper_ips, kafka_ips, zookeeper_port, kafka_port):
     instances=[]
-    for ec2_ip in req_aws_ips(service, region):
-        instances.append(Instance(name=service,ip=ec2_ip,port=port))
-    return instances
+    click.echo('using manual ip\'s ...')
+    man = ManualInstances()
+    instances += man.getIps('zookeeper',zookeeper_ips, zookeeper_port)
+    instances += man.getIps('kafka',kafka_ips, kafka_port)
+    connect(jump_host,instances)
 
-aws_filter = lambda name,value: [{'Name':'tag:'+name,'Values':[value]}]
 
-def req_aws_ips(service, region):
-    ips=[]
-    client = boto3.client('ec2')
-    response = client.describe_instances(Filters=aws_filter('Name',service))
-    for res in response.get('Reservations'):
-        for instance in res.get('Instances'):
-            ips.append(instance.get(u'PrivateIpAddress'))
-    return ips
+def connect(jump_host,instances):
+    print_instances(instances)
+    add_local_interfaces(instances)
+    connect_ssh_tunnel(jump_host,instances)
+    remove_local_interfaces(instances)
 
 def add_local_interfaces(instances):
     click.echo(' * adding interface, user password might be needed')
@@ -49,7 +55,7 @@ def add_local_interfaces(instances):
         subprocess.call(cmd)
 
 def remove_local_interfaces(instances):
-    click.echo(' * removing interface, user password might be needed')
+    click.echo(' * removing interface, user/root password might be needed')
     for instance in instances:
         if sys.platform == 'darwin':
             cmd = ['sudo', 'ifconfig', 'lo0', '-alias', instance.ip]
@@ -70,11 +76,5 @@ def connect_ssh_tunnel(jump_host,instances):
         opts += ['-L','{ip}:{port}:{ip}:{port}'.format(ip=i.ip,port=i.port)]
     subprocess.call(['ssh'] + opts + [jump_host])
 
-class Instance:
-    def __init__(self, name, ip, port):
-        self.name = name
-        self.ip = ip
-        self.port = port
-
 if __name__ == '__main__':
     cli()

setup.py

@@ -3,7 +3,7 @@
 setup(
     name='kafkatunnel',
     version='0.2',
-    py_modules=['kafkatunnel'],
+    py_modules=['kafkatunnel','Instance'],
     install_requires=[
         'Click',
         'boto3'