diff --git a/Instance.py b/Instance.py new file mode 100644 index 0000000..718c8a0 --- /dev/null +++ b/Instance.py @@ -0,0 +1,38 @@ +import boto3 + +class Instance: + def __init__(self, name, ip, port): + self.name = name + self.ip = ip + self.port = port + +class RetrieveInstanceIPs(): + def getIps(self,service,port): + raise NotImplementedError("Subclass must implement abstract method") + +class ManualInstances(RetrieveInstanceIPs): + def getIps(self,service,ips,port): + instances=[] + for ip in ips.split(','): + instances.append(Instance(name=service,ip=ip,port=port)) + return instances + +class AWSInstances(RetrieveInstanceIPs): + def __init__(self,profile,region): + self.profile = profile + self.region = region + boto3.setup_default_session(profile_name=profile) + def getIps(self,service,port): + instances=[] + for ec2_ip in self.req_aws_ips(service, self.region): + instances.append(Instance(name=service,ip=ec2_ip,port=port)) + return instances + def req_aws_ips(self,service, region): + ips=[] + aws_filter = lambda name,value: [{'Name':'tag:'+name,'Values':[value]}] + client = boto3.client('ec2') + response = client.describe_instances(Filters=aws_filter('Name',service)) + for res in response.get('Reservations'): + for instance in res.get('Instances'): + ips.append(instance.get(u'PrivateIpAddress')) + return ips diff --git a/Instance.pyc b/Instance.pyc new file mode 100644 index 0000000..fa0ca15 Binary files /dev/null and b/Instance.pyc differ diff --git a/README.md b/README.md index ad1acba..9dfc9b4 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,17 @@ Install Usage ----- - $ kafkatunnel ec2-user@awsjumphost +`kafkatool.py` supports two ways of passing the remote zookeeper/kafka ip's: + +* automatic retrival by ec2 resource tags (Name=zookeeper/kafka) + + $ kafkatunnel aws ec2-user@awsjumphost + +* manual passing the ip's + + $ kafkatunnel manual 10.11.85.128,10.11.82.30,10.11.83.9 10.11.80.7,10.11.80.123,10.11.81.13 + +afterwards you have to provide your root password in order to create the interfaces zookeeper on 10.11.85.128 port 2181 zookeeper on 10.11.82.30 port 2181 diff --git a/kafkatunnel.py b/kafkatunnel.py index 76612dc..4d1b329 100755 --- a/kafkatunnel.py +++ b/kafkatunnel.py @@ -1,43 +1,49 @@ #!/usr/bin/env python3 import click -import boto3 import sys import subprocess +from Instance import ManualInstances, AWSInstances -@click.command() +@click.group(help='Access kafka/zookeeper via ssh tunnel to consume and produce messages from your local machine') +def cli(): + pass + +@cli.command(help='retrieve kafka/zookeeper ip\'s from AWS (important: a resource tag with Name=kafka/zookeeper is needed)') @click.argument('jump_host') @click.option('-zp','--zookeeper_port',default='2181') @click.option('-kp','--kafka_port',default='9091') @click.option('-r','--region',default='ap-southeast-2') @click.option('-p','--profile',default='default') -def cli(jump_host, zookeeper_port, kafka_port, region, profile): +def aws(jump_host,zookeeper_port,kafka_port,region,profile): instances=[] - click.echo('accessing kafka & zookeeper via ssh ...') - boto3.setup_default_session(profile_name=profile) - instances += req_instances('zookeeper', zookeeper_port, region) - instances += req_instances('kafka', kafka_port, region) - print_instances(instances) - add_local_interfaces(instances) - connect_ssh_tunnel(jump_host,instances) - remove_local_interfaces(instances) + click.echo('retrieving ip\'s from AWS zookeeper/kafka ec2 instances by tag_name ...') + aws = AWSInstances(profile,region) + instances += aws.getIps('zookeeper',zookeeper_port) + instances += aws.getIps('kafka',kafka_port) + connect(jump_host,instances) + -def req_instances(service, port, region): +@cli.command(help='provide the IP\'s of your zookeeper/kafka') +@click.argument('jump_host') +@click.argument('zookeeper_ips') +@click.argument('kafka_ips') +@click.option('-zp','--zookeeper_port',default='2181') +@click.option('-kp','--kafka_port',default='9091') +def manual(jump_host,zookeeper_ips, kafka_ips, zookeeper_port, kafka_port): instances=[] - for ec2_ip in req_aws_ips(service, region): - instances.append(Instance(name=service,ip=ec2_ip,port=port)) - return instances + click.echo('using manual ip\'s ...') + man = ManualInstances() + instances += man.getIps('zookeeper',zookeeper_ips, zookeeper_port) + instances += man.getIps('kafka',kafka_ips, kafka_port) + connect(jump_host,instances) -aws_filter = lambda name,value: [{'Name':'tag:'+name,'Values':[value]}] -def req_aws_ips(service, region): - ips=[] - client = boto3.client('ec2') - response = client.describe_instances(Filters=aws_filter('Name',service)) - for res in response.get('Reservations'): - for instance in res.get('Instances'): - ips.append(instance.get(u'PrivateIpAddress')) - return ips +def connect(jump_host,instances): + print_instances(instances) + add_local_interfaces(instances) + connect_ssh_tunnel(jump_host,instances) + remove_local_interfaces(instances) def add_local_interfaces(instances): click.echo(' * adding interface, user password might be needed') @@ -49,7 +55,7 @@ def add_local_interfaces(instances): subprocess.call(cmd) def remove_local_interfaces(instances): - click.echo(' * removing interface, user password might be needed') + click.echo(' * removing interface, user/root password might be needed') for instance in instances: if sys.platform == 'darwin': cmd = ['sudo', 'ifconfig', 'lo0', '-alias', instance.ip] @@ -70,11 +76,5 @@ def connect_ssh_tunnel(jump_host,instances): opts += ['-L','{ip}:{port}:{ip}:{port}'.format(ip=i.ip,port=i.port)] subprocess.call(['ssh'] + opts + [jump_host]) -class Instance: - def __init__(self, name, ip, port): - self.name = name - self.ip = ip - self.port = port - if __name__ == '__main__': cli() diff --git a/kafkatunnel.pyc b/kafkatunnel.pyc new file mode 100644 index 0000000..3d3898a Binary files /dev/null and b/kafkatunnel.pyc differ diff --git a/setup.py b/setup.py index 0801acb..2c37e61 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='kafkatunnel', version='0.2', - py_modules=['kafkatunnel'], + py_modules=['kafkatunnel','Instance'], install_requires=[ 'Click', 'boto3'