Permalink
Browse files

Added the -e/--expand option to hostlist.

  • Loading branch information...
1 parent ef272d9 commit 9fcf5acd09872d9374c111da79140dd43909a611 @dwighthubbard committed Aug 15, 2012
View
@@ -39,11 +39,12 @@ def _get_plugins():
""" Find all the hostlists plugins """
plugins=global_plugins
pluginlist=[]
- for directory in sys.path:
- if os.path.isdir(os.path.join(directory,'hostlists_plugins')):
- templist=os.listdir(os.path.join(directory,'hostlists_plugins'))
+ plugin_path=['/lib/hostlists']+sys.path
+ for directory in plugin_path:
+ if os.path.isdir(os.path.join(directory,'plugins')):
+ templist=os.listdir(os.path.join(directory,'plugins'))
for item in templist:
- pluginlist.append(os.path.join(os.path.join(directory,'hostlists_plugins'),item))
+ pluginlist.append(os.path.join(os.path.join(directory,'plugins'),item))
pluginlist.sort()
# Create a dict mapping the plugin name to the plugin method
for item in pluginlist:
@@ -96,7 +97,11 @@ def compress(range_list):
parser = optparse.OptionParser(usage="usage: %prog [options] plugin:parameters")
parser.add_option("-s","--sep",dest="sep",default=',',help="Seperator character, default=\",\"")
parser.add_option("--onepass",dest="onepass",default=False,action="store_true")
+ parser.add_option("--expand","-e",dest="expand",default=False,action="store_true",help="Expand the host list and dislay one host per line")
(options, args) = parser.parse_args()
range=','.join(args)
- print compress(expand(range.split(',')))
+ if options.expand:
+ print '\n'.join(expand(range.split(',')))
+ else:
+ print compress(expand(range.split(',')))
View
@@ -1,22 +1,36 @@
#!/usr/bin/env python
""" hostlists plugin to get hosts from dns """
import dns.resolver
-
+import dns.reversename
def name():
return 'dns'
def expand(value):
tmplist=[]
- adresses=[]
+ addresses=[]
try:
- answers = dns.resolver.query(value, 'A')
+ answers = list(dns.resolver.query(value))
except dns.resolver.NoAnswer:
answers=[]
- try:
- answers = answers+dns.resolver.query(value,'CNAME')
- except dns.resolver.NoAnswer:
- pass
for rdata in answers:
- tmplist.append(rdata.address)
+ addresses.append(rdata.address)
+ for address in addresses:
+ result=dns.reversename.from_address(address)
+ try:
+ # See if we can reverse resolv the IP address and insert
+ # it only if it's a unique hostname.
+ revaddress=str(dns.resolver.query(result,'PTR')[0]).strip('.')
+ if revaddress not in tmplist:
+ tmplist.append(revaddress)
+ except dns.resolver.NXDOMAIN:
+ tmplist.append(address)
+ # if the tmplist with the reverse resolved hostnames
+ # is not the same length as the list of addresses then
+ # the site has the same hostname assigned to more than
+ # one IP address dns reverse resolving
+ # If this happens we return the IP addresses because
+ # they are unique, otherwise we return the hostnames.
+ if len(tmplist) < len(addresses):
+ return addresses
return tmplist
View
Binary file not shown.
@@ -57,4 +57,7 @@ def expand_item(item):
block+=letter
elif not in_block:
pre_block += letter
- return result
+ if len(result):
+ return result
+ else:
+ return [item]
View
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
import os
from distutils.core import setup
"""
@@ -18,13 +18,14 @@
setup(
name="sshmap",
- version="0.2.42",
+ version="0.2.99",
author="Dwight Hubbard",
author_email="dhubbard@yahoo-inc.com",
url="http://www.yahoo.com",
license="LICENSE.txt",
- packages=["sshmap","hostlists","hostlists_plugins"],
- scripts=["sshmap/sshmap",'hostlists/hostlists'],
+ packages=["sshmap","hostlists"],
+ data_files=[('/lib/hostlists/plugins',['hostlists_plugins/files.py','hostlists_plugins/dns.py','hostlists_plugins/dnsip.py','hostlists_plugins/range.py'])],
+ scripts=["sshmap/sshmap.py",'hostlists/hostlists'],
long_description=open('README.txt').read(),
description="A SSH Multiplexer designed to use ssh to perform map/reduce like operations",
requires=['paramiko'],
View
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
"""
Copyright (c) 2010 Yahoo! Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,5 +14,3 @@
limitations under the License. See accompanying LICENSE file.
"""
from sshmap import *
-
-
View
@@ -1,138 +0,0 @@
-#!/usr/bin/env python
-""" Python based ssh multiplexer optimized for map operations """
-
-"""
- Copyright (c) 2012 Yahoo! Inc. All rights reserved.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
-"""
-
-""" Filter callback handlers """
-def flowthrough(result):
- """
- Builtin Callback, return the raw data passed
-
- >>> result=callback_flowthrough(ssh_result(["output"], ["error"],"foo", 0))
- >>> result.dump()
- foo output error 0 0 None
- """
- return result
-
-def summarize_failures(result):
- """
- Builtin Callback, put a summary of failures into parm
- """
- failures=result.setting('failures')
- if not failures:
- result.parm['failures']=[]
- failures=[]
- if result.ssh_retcode:
- failures.append(result.host)
- result.parm['failures']=failures
- return result
-
-def exec_command(result):
- """
- Builtin Callback, pass the results to a command/script
- """
- script=result.setting("callback_script")
- if not script:
- return result
- status_clear()
- result_out,result_err=subprocess.Popen(script + " "+result.host, shell=True,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate(result.out_string()+result.err_string())
- result.out=[result_out]
- result.err=[result_err]
- print result.out_string()
- return result
-
-def filter_match(result):
- """
- Builtin Callback, remove all output if the string is not found in the output
- similar to grep
- """
- if result.out_string().find(result.setting('match')) == -1 and result.err_string().find(result.setting('match')) == -1:
- result.out=''
- result.err=''
- return result
-
-def filter_json(result):
- """
- Builtin Callback, change stdout to json
-
- >>> result=callback_filter_json(ssh_result(["output"], ["error"],"foo", 0))
- >>> result.dump()
- foo [["output"], ["error"], 0] error 0 0 None
- """
- result.out=[json.dumps((result.out,result.err,result.retcode))]
- return result
-
-def filter_base64(result):
- """
- Builtin Callback, base64 encode the info in out and err streams
- """
- result.out=[base64.b64encode(result.out_string)]
- result.err=[base64.b64encode(result.err_string)]
- return result
-
-""" Status callback handlers """
-def status_count(result):
- """
- Builtin Callback, show the count complete/remaining
- """
- # The master process inserts the status into the
- # total_host_count and completed_host_count variables
- sys.stderr.write('\x1b[0G\x1b[0K%s/%s'%(result.setting('completed_host_count'),result.setting('total_host_count')))
- sys.stderr.flush()
- return result
-
-""" Output callback handlers """
-def output_prefix_host(result):
- """
- Builtin Callback, print the output with the hostname: prefixed to each line
-
- >>> result=callback_output_prefix_host(ssh_result(['out'],['err'], 'hostname', 0))
- hostname: out
- >>> result.dump()
- hostname hostname: out hostname: Error: err 0 0 None
- """
- output=[]
- error=[]
- status_clear()
- # If summarize_failures option is set don't print ssh errors inline
- if result.setting('summarize_failed') and result.ssh_retcode:
- return result
- if result.setting('print_rc'):
- rc=' SSH_Returncode: %d\tCommand_Returncode: %d' % (result.ssh_retcode,result.retcode)
- else:
- rc=''
- if result.ssh_retcode:
- print >> sys.stderr, '%s: %s' % (result.host,result.ssh_error_message())
- error=['%s: %s' % (result.host,result.ssh_error_message())]
- if len(result.out_string()):
- for line in result.out:
- if line:
- print '%s:%s %s' % (result.host,rc,line.strip())
- output.append('%s:%s %s\n' % (result.host,rc,line.strip()))
- if len(result.err_string()):
- for line in result.err:
- if line:
- print >> sys.stderr, '%s:%s %s' % (result.host,rc,line.strip())
- error.append('%s:%s Error: %s\n' % (result.host,rc,line.strip()))
- if result.setting('output'):
- if not len(result.out_string()) and not len(result.err_string()) and not result.setting('only_output') and result.setting('print_rc'):
- print '%s:%s' % (result.host,rc)
- sys.stdout.flush()
- sys.stderr.flush()
- result.out=output
- result.err=error
- return result
-
View
View
@@ -113,9 +113,15 @@ def setting(self,key):
def ssh_error_message(self):
""" Return the ssh_error_message for the error code """
return RUN_CODES[self.ssh_retcode]
- def dump(self):
+ def dump(self,return_parm=True,return_retcode=True):
""" Print all our public values """
- print self.host,self.out_string().replace('\n',''),self.err_string().replace('\n',''),self.retcode,self.ssh_retcode,self.parm
+ print self.host,self.out_string().replace('\n',''),self.err_string().replace('\n',''),
+ if return_retcode:
+ print self.retcode,
+ if return_parm:
+ print self.ssh_retcode,self.parm
+ else:
+ print
def print_output(self):
""" Print output from the commands """
for line in self.out:
@@ -134,11 +140,15 @@ class ssh_results(list):
def dump(self):
""" Dump all the result objects """
for item in self.__iter__():
- item.dump()
- def print_output(self):
+ item.dump(return_parm=False,return_retcode=False)
+ print self.parm
+ def print_output(self,summarize_failures=False):
""" Print all the objects """
for item in self.__iter__():
item.print_output()
+ if summarize_failures:
+ if len(self.parm['failures']):
+ print 'SSH Failures:',','.join(self.parm['failures']).strip(',')
class fastSSHClient(paramiko.SSHClient):
""" Paramiko SSHClient class extended with timeout support """
@@ -153,7 +163,7 @@ def exec_command(self, command, bufsize=-1, timeout=None):
def run_command(host,command="uname -a",username=None,password=None,sudo=False,script=None,timeout=None,parms=None,client=None,bufsize=-1,cwd='/tmp',logging=False):
"""
- Run a command or script on a remote node via ssh
+ #Run a command or script on a remote node via ssh
"""
# Guess any parameters not passed that can be
if isinstance(host,types.TupleType):
@@ -261,29 +271,7 @@ def status_clear():
sys.stderr.write('\x1b[0G\x1b[0K')
sys.stderr.flush()
-def read_conf(key=None,prompt=True):
- """ Read settings from the config file """
- try:
- conf=json.load(open(os.path.expanduser('~/.fastssh2.conf'),'r'))
- except IOError:
- conf=conf_defaults
- if key:
- try:
- return conf[key].encode('ascii')
- except KeyError:
- pass
- else:
- return conf
- if key and prompt:
- conf[key]=raw_input(conf_desc[key]+': ')
- fh=open(os.path.expanduser('~/.fastssh2.conf'),'w')
- os.fchmod(fh.fileno(),stat.S_IRUSR|stat.S_IWUSR)
- json.dump(conf,fh)
- fh.close()
- return conf[key]
- else:
- return None
-
+# Built in callbacks
""" Filter callback handlers """
def callback_flowthrough(result):
"""
@@ -404,6 +392,30 @@ def callback_output_prefix_host(result):
result.err=error
return result
+def read_conf(key=None,prompt=True):
+ """ Read settings from the config file """
+ try:
+ conf=json.load(open(os.path.expanduser('~/.fastssh2.conf'),'r'))
+ except IOError:
+ conf=conf_defaults
+ if key:
+ try:
+ return conf[key].encode('ascii')
+ except KeyError:
+ pass
+ else:
+ return conf
+ if key and prompt:
+ conf[key]=raw_input(conf_desc[key]+': ')
+ fh=open(os.path.expanduser('~/.fastssh2.conf'),'w')
+ os.fchmod(fh.fileno(),stat.S_IRUSR|stat.S_IWUSR)
+ json.dump(conf,fh)
+ fh.close()
+ return conf[key]
+ else:
+ return None
+
+
def init_worker():
""" Set up the signal handler for new worker threads """
signal.signal(signal.SIGINT, signal.SIG_IGN)
@@ -449,15 +461,12 @@ def run(range,command,username=None,password=None,sudo=False,script=None,timeout
map_command=pool.imap_unordered
# Create a process pool and pass the parameters to it
- #status_info(output_callback,' \bSpawning processes')
try:
for result in map_command(run_command,[(host,command,username,password,sudo,script,timeout,results.parm,client) for host in hosts]):
- #status_info(output_callback,' \bLoop start')
results.parm['completed_host_count']=results.parm['completed_host_count']+1
result.parm=results.parm
if isinstance(output_callback,types.ListType):
for callback in output_callback:
- #print 'callback',callback,len(multiprocessing.active_children())
result=callback(result)
else:
result=output_callback(result)

0 comments on commit 9fcf5ac

Please sign in to comment.