From 014714336e0c554ab3decc14814be0fdfca4babe Mon Sep 17 00:00:00 2001 From: dhubbard Date: Mon, 8 Jul 2013 17:51:36 -0700 Subject: [PATCH 01/52] More PEP8 cleanups and removal of junk comments --- sshmap/sshmap.py | 141 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 98 insertions(+), 43 deletions(-) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index b407cae..a6c326f 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -214,6 +214,14 @@ class fastSSHClient(ssh.SSHClient): """ ssh SSHClient class extended with timeout support """ def exec_command(self, command, bufsize=-1, timeout=None, pty=False): + """ + Execute a command + :param command: + :param bufsize: + :param timeout: + :param pty: + :return: + """ chan = self._transport.open_session() chan.settimeout(timeout) if pty: @@ -226,30 +234,36 @@ def exec_command(self, command, bufsize=-1, timeout=None, pty=False): def _term_readline(handle): - #print '_iterm_readline' - #print type(handle) char = handle.read(1) - #print '%s' % (char), type(char) buf = "" - #print '_iterm_readline: starting loop' try: while char: - #print '_item_readline: appending',type(buf),type(char) buf += char if char in ['\r', '\n']: - #print '_iterm_readline - Found line', len(buf), char, buf return buf char = handle.read(1) except Exception, message: print Exception, message - #print '_item_readline - Exit', buf return buf -def run_command(host, command="uname -a", username=None, password=None, sudo=False, script=None, timeout=None, - parms=None, client=None, bufsize=-1, cwd='/tmp', logging=False): +def run_command(host, command="uname -a", username=None, password=None, + sudo=False, script=None, timeout=None, parms=None, client=None, + bufsize=-1, cwd='/tmp', logging=False): """ Run a command or script on a remote node via ssh + :param host: + :param command: + :param username: + :param password: + :param sudo: + :param script: + :param timeout: + :param parms: + :param client: + :param bufsize: + :param cwd: + :param logging: """ # Guess any parameters not passed that can be if isinstance(host, types.TupleType): @@ -290,7 +304,8 @@ def run_command(host, command="uname -a", username=None, password=None, sudo=Fal close_client = True # noinspection PyBroadException try: - client.connect(host, username=username, password=password, timeout=timeout) + client.connect(host, username=username, password=password, + timeout=timeout) except ssh.AuthenticationException: result.ssh_retcode = RUN_FAIL_AUTH return result @@ -307,15 +322,19 @@ def run_command(host, command="uname -a", username=None, password=None, sudo=Fal result.ssh_retcode = RUN_FAIL_UNKNOWN return result try: - # We have to force a sudo -k first or we can't reliably know we'll be prompted for our password + # We have to force a sudo -k first or we can't reliably know we'll be + # prompted for our password if sudo: - stdin, stdout, stderr, chan = client.exec_command('sudo -k %s' % command, timeout=timeout, bufsize=bufsize, - pty=True) + stdin, stdout, stderr, chan = client.exec_command( + 'sudo -k -S %s' % command, + timeout=timeout, bufsize=bufsize, pty=True + ) if not chan: result.ssh_retcode = RUN_FAIL_CONNECT return result else: - stdin, stdout, stderr, chan = client.exec_command(command, timeout=timeout, bufsize=bufsize) + stdin, stdout, stderr, chan = client.exec_command( + command, timeout=timeout, bufsize=bufsize) if not chan: result.ssh_retcode = RUN_FAIL_CONNECT result.err = ["WTF, this shouldn't happen\n"] @@ -335,7 +354,8 @@ def run_command(host, command="uname -a", username=None, password=None, sudo=Fal seen_password = False seen_password_prompt = False #print 'READ:',prompt - while 'assword:' in prompt or password in prompt or 'try again' in prompt or len(prompt.strip()) == 0: + while 'assword:' in prompt or password in prompt or \ + 'try again' in prompt or len(prompt.strip()) == 0: if 'try again' in prompt: result.ssh_retcode = RUN_FAIL_BADPASSWORD return result @@ -352,8 +372,8 @@ def run_command(host, command="uname -a", username=None, password=None, sudo=Fal result.ssh_retcode = RUN_FAIL_TIMEOUT return result if script: - # Pass the script over stdin and close the channel so the receving end gets an EOF - # process it as a django template with the arguments passed + # Pass the script over stdin and close the channel so the receving end + # gets an EOF process it as a django template with the arguments passed # noinspection PyBroadException try: import django.template @@ -367,7 +387,7 @@ def run_command(host, command="uname -a", username=None, password=None, sudo=Fal else: c = django.template.Context({ }) stdin.write(django.template.Template(template).render(c)) - except Exception, e: + except Exception as e: stdin.write(open(script, 'r').read()) stdin.flush() stdin.channel.shutdown_write() @@ -457,12 +477,18 @@ def callback_exec_command(result): if not script: return result status_clear() - result_out, result_err = subprocess.Popen(script + " " + result.host, shell = True, stdin = subprocess.PIPE, - stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate( - result.out_string() + result.err_string()) + result_out, result_err = subprocess.Popen( + script + " " + result.host, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ).communicate( + result.out_string() + result.err_string() + ) result.out = [result_out] result.err = [result_err] - print result.out_string() + print(result.out_string()) return result @@ -500,11 +526,13 @@ def callback_aggregate_output(result): def callback_filter_match(result): """ - Builtin Callback, remove all output if the string is not found in the output + Builtin Callback, remove all output if the string is not found in the + output similar to grep + :param result: """ - if result.out_string().find(result.setting('match')) == -1 and result.err_string().find( - result.setting('match')) == -1: + if result.out_string().find(result.setting('match')) == -1 and \ + result.err_string().find(result.setting('match')) == -1: result.out = '' result.err = '' return result @@ -540,7 +568,8 @@ def callback_status_count(result): # The master process inserts the status into the # total_host_count and completed_host_count variables sys.stderr.write('\x1b[0G\x1b[0K%s/%s' % ( - result.setting('completed_host_count'), result.setting('total_host_count'))) + result.setting('completed_host_count'), + result.setting('total_host_count'))) sys.stderr.flush() return result @@ -563,7 +592,8 @@ def callback_output_prefix_host(result): if result.setting('summarize_failed') and result.ssh_retcode: return result if result.setting('print_rc'): - rc = ' SSH_Returncode: %d\tCommand_Returncode: %d' % (result.ssh_retcode, result.retcode) + rc = ' SSH_Returncode: %d\tCommand_Returncode: %d' % ( + result.ssh_retcode, result.retcode) else: rc = '' if result.ssh_retcode: @@ -619,15 +649,32 @@ def init_worker(): signal.signal(signal.SIGINT, signal.SIG_IGN) -def run(host_range, command, username=None, password=None, sudo=False, script=None, timeout=None, sort=False, - bufsize=-1, cwd='/tmp', jobs=None, output_callback=callback_summarize_failures, parms=None, shuffle=False, - chunksize=None): +def run(host_range, command, username=None, password=None, sudo=False, + script=None, timeout=None, sort=False, bufsize=-1, cwd='/tmp', + jobs=None, output_callback=callback_summarize_failures, parms=None, + shuffle=False, chunksize=None): """ Run a command on a hostlists host_range of hosts + :param host_range: + :param command: + :param username: + :param password: + :param sudo: + :param script: + :param timeout: + :param sort: + :param bufsize: + :param cwd: + :param jobs: + :param output_callback: + :param parms: + :param shuffle: + :param chunksize: + >>> res=run(host_range='localhost',command="echo ok") - >>> print res[0].dump() - localhost ok 0 0 {'failures': [], 'total_host_count': 1, 'completed_host_count': 1} - None + >>> print(res[0].dump()) + localhost ok 0 0 {'failures': [], 'total_host_count': 1, + 'completed_host_count': 1} """ status_info(output_callback, 'Looking up hosts') hosts = hostlists.expand(hostlists.range_split(host_range)) @@ -673,7 +720,7 @@ def run(host_range, command, username=None, password=None, sudo=False, script=No if jobs > len(hosts): jobs = len(hosts) - pool = multiprocessing.Pool(processes = jobs, initializer = init_worker) + pool = multiprocessing.Pool(processes=jobs, initializer=init_worker) if not chunksize: chunksize = 1 if jobs >= len(hosts): @@ -692,21 +739,28 @@ def run(host_range, command, username=None, password=None, sudo=False, script=No else: map_command = pool.imap_unordered - if isinstance(output_callback, types.ListType) and callback_status_count in output_callback: + if isinstance(output_callback, types.ListType) and \ + callback_status_count in output_callback: callback_status_count(ssh_result(parm=results.parm)) # Create a process pool and pass the parameters to it status_clear() - status_info(output_callback, 'Sending %d commands to each process' % chunksize) + status_info( + output_callback, 'Sending %d commands to each process' % chunksize) if callback_status_count in output_callback: callback_status_count(ssh_result(parm=results.parm)) try: - for result in map_command(run_command, - [(host, command, username, password, sudo, script, timeout, results.parm, client) for - host in hosts], chunksize): - #results.parm['active_processes']=len(multiprocessing.active_children()) + for result in map_command( + run_command, + [ + ( + host, command, username, password, sudo, script, timeout, + results.parm, client + ) for host in hosts + ], + chunksize): results.parm['completed_host_count'] += 1 result.parm = results.parm if isinstance(output_callback, types.ListType): @@ -718,13 +772,14 @@ def run(host_range, command, username=None, password=None, sudo=False, script=No results.append(result) pool.close() except KeyboardInterrupt: - print 'ctrl-c pressed' + print('ctrl-c pressed') pool.terminate() - #except Exception,e: + #except Exception as e: # print 'unknown error encountered',Exception,e # pass pool.terminate() - if isinstance(output_callback, types.ListType) and callback_status_count in output_callback: + if isinstance(output_callback, types.ListType) and \ + callback_status_count in output_callback: status_clear() return results From d4e20bafb6e764e82c7710476c22716261088e3b Mon Sep 17 00:00:00 2001 From: dhubbard Date: Mon, 8 Jul 2013 18:38:43 -0700 Subject: [PATCH 02/52] Split out callbacks again --- sshmap/callback.py | 227 +++++++++++++++++++++++++++++++++++++++++++++ sshmap/sshmap.py | 226 ++------------------------------------------ 2 files changed, 237 insertions(+), 216 deletions(-) create mode 100644 sshmap/callback.py diff --git a/sshmap/callback.py b/sshmap/callback.py new file mode 100644 index 0000000..a0d8cae --- /dev/null +++ b/sshmap/callback.py @@ -0,0 +1,227 @@ +#Copyright (c) 2012 Yahoo! Inc. All rights reserved. +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +__author__ = 'dhubbard' +""" +sshmap built in callback handlers +""" +import os +import sys +import hashlib +import json +import stat +import base64 +import subprocess + +import sshmap + +# Filter callback handlers +def flowthrough(result): + """ + Builtin Callback, return the raw data passed + + >>> result=flowthrough(ssh_result(["output"], ["error"],"foo", 0)) + >>> result.dump() + foo output error 0 0 None + """ + return result + + +def summarize_failures(result): + """ + Builtin Callback, put a summary of failures into parm + """ + failures = result.setting('failures') + if not failures: + result.parm['failures'] = [] + failures = [] + if result.ssh_retcode: + failures.append(result.host) + result.parm['failures'] = failures + return result + + +def exec_command(result): + """ + Builtin Callback, pass the results to a command/script + :param result: + """ + script = result.setting("callback_script") + if not script: + return result + sshmap.status_clear() + result_out, result_err = subprocess.Popen( + script + " " + result.host, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ).communicate( + result.out_string() + result.err_string() + ) + result.out = [result_out] + result.err = [result_err] + print(result.out_string()) + return result + + +def aggregate_output(result): + """ Builtin Callback, Aggregate identical results """ + aggregate_hosts = result.setting('aggregate_hosts') + if not aggregate_hosts: + aggregate_hosts = {} + collapsed_output = result.setting('collapsed_output') + if not collapsed_output: + collapsed_output = {} + h = hashlib.md5() + h.update(result.out_string()) + h.update(result.err_string()) + if result.ssh_retcode: + h.update(result.ssh_error_message()) + digest = h.hexdigest() + if digest in aggregate_hosts.keys(): + aggregate_hosts[digest].append(result.host) + else: + aggregate_hosts[digest] = [result.host] + if result.ssh_retcode: + error = [] + if result.err: + error = result.err + error.append(result.ssh_error_message()) + collapsed_output[digest] = (result.out, error) + else: + collapsed_output[digest] = (result.out, result.err) + result.parm['aggregate_hosts'] = aggregate_hosts + if collapsed_output: + result.parm['collapsed_output'] = collapsed_output + return result + + +def filter_match(result): + """ + Builtin Callback, remove all output if the string is not found in the + output + similar to grep + :param result: + """ + if result.out_string().find(result.setting('match')) == -1 and \ + result.err_string().find(result.setting('match')) == -1: + result.out = '' + result.err = '' + return result + + +def filter_json(result): + """ + Builtin Callback, change stdout to json + + >>> result=filter_json(ssh_result(["output"], ["error"],"foo", 0)) + >>> result.dump() + foo [["output"], ["error"], 0] error 0 0 None + """ + result.out = [json.dumps((result.out, result.err, result.retcode))] + return result + + +def filter_base64(result): + """ + Builtin Callback, base64 encode the info in out and err streams + """ + result.out = [base64.b64encode(result.out_string)] + result.err = [base64.b64encode(result.err_string)] + return result + + +#Status callback handlers +def status_count(result): + """ + Builtin Callback, show the count complete/remaining + :param result: + """ + # The master process inserts the status into the + # total_host_count and completed_host_count variables + sys.stderr.write('\x1b[0G\x1b[0K%s/%s' % ( + result.setting('completed_host_count'), + result.setting('total_host_count'))) + sys.stderr.flush() + return result + + +#Output callback handlers +def output_prefix_host(result): + """ + Builtin Callback, print the output with the hostname: prefixed to each line + :param result: + hostname: out + + >>> result=sshmap.callback.output_prefix_host(ssh_result(['out'],['err'], 'hostname', 0)) + >>> result.dump() + """ + output = [] + error = [] + sshmap.status_clear() + # If summarize_failures option is set don't print ssh errors inline + if result.setting('summarize_failed') and result.ssh_retcode: + return result + if result.setting('print_rc'): + rc = ' SSH_Returncode: %d\tCommand_Returncode: %d' % ( + result.ssh_retcode, result.retcode) + else: + rc = '' + if result.ssh_retcode: + print >> sys.stderr, '%s: %s' % (result.host, result.ssh_error_message()) + error = ['%s: %s' % (result.host, result.ssh_error_message())] + if len(result.out_string()): + for line in result.out: + if line: + print '%s:%s %s' % (result.host, rc, line.strip()) + output.append('%s:%s %s\n' % (result.host, rc, line.strip())) + if len(result.err_string()): + for line in result.err: + if line: + print >> sys.stderr, '%s:%s %s' % (result.host, rc, line.strip()) + error.append('%s:%s Error: %s\n' % (result.host, rc, line.strip())) + if result.setting('output'): + if not len(result.out_string()) and not len(result.err_string()) and not result.setting( + 'only_output') and result.setting('print_rc'): + print '%s:%s' % (result.host, rc) + sys.stdout.flush() + sys.stderr.flush() + result.out = output + result.err = error + return result + + +def read_conf(key=None, prompt=True): + """ Read settings from the config file + :param key: + :param prompt: + """ + try: + conf = json.load(open(os.path.expanduser('~/.sshmap.conf'), 'r')) + except IOError: + conf = sshmap.conf_defaults + if key: + try: + return conf[key].encode('ascii') + except KeyError: + pass + else: + return conf + if key and prompt: + conf[key] = raw_input(sshmap.conf_desc[key] + ': ') + fh = open(os.path.expanduser('~/.sshmap2.conf'), 'w') + os.fchmod(fh.fileno(), stat.S_IRUSR | stat.S_IWUSR) + json.dump(conf, fh) + fh.close() + return conf[key] + else: + return None diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index a6c326f..203bf78 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -22,17 +22,12 @@ # Python Standard Library imports import sys import os -import stat import getpass import socket import types -import base64 import random import signal -import hashlib -import json import multiprocessing -import subprocess import logging # Imports from external python extension modules @@ -40,6 +35,7 @@ # Imports from other sshmap modules import hostlists +import sshmap.callback # Defaults JOB_MAX = 100 @@ -427,7 +423,8 @@ def status_info(callbacks, text): """ #print callbacks,text #return - if isinstance(callbacks, list) and callback_status_count in callbacks: + if isinstance(callbacks, list) and \ + sshmap.callback.status_count in callbacks: status_clear() sys.stderr.write(text) sys.stderr.flush() @@ -441,209 +438,6 @@ def status_clear(): #sys.stderr.flush() -# Built in callbacks -# Filter callback handlers -def callback_flowthrough(result): - """ - Builtin Callback, return the raw data passed - - >>> result=callback_flowthrough(ssh_result(["output"], ["error"],"foo", 0)) - >>> result.dump() - foo output error 0 0 None - """ - return result - - -def callback_summarize_failures(result): - """ - Builtin Callback, put a summary of failures into parm - """ - failures = result.setting('failures') - if not failures: - result.parm['failures'] = [] - failures = [] - if result.ssh_retcode: - failures.append(result.host) - result.parm['failures'] = failures - return result - - -def callback_exec_command(result): - """ - Builtin Callback, pass the results to a command/script - :param result: - """ - script = result.setting("callback_script") - if not script: - return result - status_clear() - result_out, result_err = subprocess.Popen( - script + " " + result.host, - shell=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ).communicate( - result.out_string() + result.err_string() - ) - result.out = [result_out] - result.err = [result_err] - print(result.out_string()) - return result - - -def callback_aggregate_output(result): - """ Builtin Callback, Aggregate identical results """ - aggregate_hosts = result.setting('aggregate_hosts') - if not aggregate_hosts: - aggregate_hosts = {} - collapsed_output = result.setting('collapsed_output') - if not collapsed_output: - collapsed_output = {} - h = hashlib.md5() - h.update(result.out_string()) - h.update(result.err_string()) - if result.ssh_retcode: - h.update(result.ssh_error_message()) - digest = h.hexdigest() - if digest in aggregate_hosts.keys(): - aggregate_hosts[digest].append(result.host) - else: - aggregate_hosts[digest] = [result.host] - if result.ssh_retcode: - error = [] - if result.err: - error = result.err - error.append(result.ssh_error_message()) - collapsed_output[digest] = (result.out, error) - else: - collapsed_output[digest] = (result.out, result.err) - result.parm['aggregate_hosts'] = aggregate_hosts - if collapsed_output: - result.parm['collapsed_output'] = collapsed_output - return result - - -def callback_filter_match(result): - """ - Builtin Callback, remove all output if the string is not found in the - output - similar to grep - :param result: - """ - if result.out_string().find(result.setting('match')) == -1 and \ - result.err_string().find(result.setting('match')) == -1: - result.out = '' - result.err = '' - return result - - -def callback_filter_json(result): - """ - Builtin Callback, change stdout to json - - >>> result=callback_filter_json(ssh_result(["output"], ["error"],"foo", 0)) - >>> result.dump() - foo [["output"], ["error"], 0] error 0 0 None - """ - result.out = [json.dumps((result.out, result.err, result.retcode))] - return result - - -def callback_filter_base64(result): - """ - Builtin Callback, base64 encode the info in out and err streams - """ - result.out = [base64.b64encode(result.out_string)] - result.err = [base64.b64encode(result.err_string)] - return result - - -#Status callback handlers -def callback_status_count(result): - """ - Builtin Callback, show the count complete/remaining - :param result: - """ - # The master process inserts the status into the - # total_host_count and completed_host_count variables - sys.stderr.write('\x1b[0G\x1b[0K%s/%s' % ( - result.setting('completed_host_count'), - result.setting('total_host_count'))) - sys.stderr.flush() - return result - - -#Output callback handlers -def callback_output_prefix_host(result): - """ - Builtin Callback, print the output with the hostname: prefixed to each line - :param result: - - >>> result=callback_output_prefix_host(ssh_result(['out'],['err'], 'hostname', 0)) - hostname: out - >>> result.dump() - hostname hostname: out hostname: Error: err 0 0 None - """ - output = [] - error = [] - status_clear() - # If summarize_failures option is set don't print ssh errors inline - if result.setting('summarize_failed') and result.ssh_retcode: - return result - if result.setting('print_rc'): - rc = ' SSH_Returncode: %d\tCommand_Returncode: %d' % ( - result.ssh_retcode, result.retcode) - else: - rc = '' - if result.ssh_retcode: - print >> sys.stderr, '%s: %s' % (result.host, result.ssh_error_message()) - error = ['%s: %s' % (result.host, result.ssh_error_message())] - if len(result.out_string()): - for line in result.out: - if line: - print '%s:%s %s' % (result.host, rc, line.strip()) - output.append('%s:%s %s\n' % (result.host, rc, line.strip())) - if len(result.err_string()): - for line in result.err: - if line: - print >> sys.stderr, '%s:%s %s' % (result.host, rc, line.strip()) - error.append('%s:%s Error: %s\n' % (result.host, rc, line.strip())) - if result.setting('output'): - if not len(result.out_string()) and not len(result.err_string()) and not result.setting( - 'only_output') and result.setting('print_rc'): - print '%s:%s' % (result.host, rc) - sys.stdout.flush() - sys.stderr.flush() - result.out = output - result.err = error - return result - - -def read_conf(key=None, prompt=True): - """ Read settings from the config file """ - try: - conf = json.load(open(os.path.expanduser('~/.sshmap.conf'), 'r')) - except IOError: - conf = conf_defaults - if key: - try: - return conf[key].encode('ascii') - except KeyError: - pass - else: - return conf - if key and prompt: - conf[key] = raw_input(conf_desc[key] + ': ') - fh = open(os.path.expanduser('~/.sshmap2.conf'), 'w') - os.fchmod(fh.fileno(), stat.S_IRUSR | stat.S_IWUSR) - json.dump(conf, fh) - fh.close() - return conf[key] - else: - return None - - def init_worker(): """ Set up the signal handler for new worker threads """ signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -651,8 +445,8 @@ def init_worker(): def run(host_range, command, username=None, password=None, sudo=False, script=None, timeout=None, sort=False, bufsize=-1, cwd='/tmp', - jobs=None, output_callback=callback_summarize_failures, parms=None, - shuffle=False, chunksize=None): + jobs=None, output_callback=sshmap.callback.summarize_failures, + parms=None, shuffle=False, chunksize=None): """ Run a command on a hostlists host_range of hosts :param host_range: @@ -740,16 +534,16 @@ def run(host_range, command, username=None, password=None, sudo=False, map_command = pool.imap_unordered if isinstance(output_callback, types.ListType) and \ - callback_status_count in output_callback: - callback_status_count(ssh_result(parm=results.parm)) + sshmap.callback.status_count in output_callback: + sshmap.callback.status_count(ssh_result(parm=results.parm)) # Create a process pool and pass the parameters to it status_clear() status_info( output_callback, 'Sending %d commands to each process' % chunksize) - if callback_status_count in output_callback: - callback_status_count(ssh_result(parm=results.parm)) + if sshmap.callback.status_count in output_callback: + sshmap.callback.status_count(ssh_result(parm=results.parm)) try: for result in map_command( @@ -779,7 +573,7 @@ def run(host_range, command, username=None, password=None, sudo=False, # pass pool.terminate() if isinstance(output_callback, types.ListType) and \ - callback_status_count in output_callback: + sshmap.callback.status_count in output_callback: status_clear() return results From f434a9325cdff1e7b6f591b42a9402986e2b3bb1 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Mon, 8 Jul 2013 18:44:41 -0700 Subject: [PATCH 03/52] Moved utility functions to their own file --- sshmap/callback.py | 6 +++-- sshmap/sshmap.py | 61 +++++++++++----------------------------------- sshmap/utility.py | 53 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 49 deletions(-) create mode 100644 sshmap/utility.py diff --git a/sshmap/callback.py b/sshmap/callback.py index a0d8cae..4025eb0 100644 --- a/sshmap/callback.py +++ b/sshmap/callback.py @@ -9,6 +9,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. +from sshmap.utility import status_clear + __author__ = 'dhubbard' """ sshmap built in callback handlers @@ -57,7 +59,7 @@ def exec_command(result): script = result.setting("callback_script") if not script: return result - sshmap.status_clear() + status_clear() result_out, result_err = subprocess.Popen( script + " " + result.host, shell=True, @@ -167,7 +169,7 @@ def output_prefix_host(result): """ output = [] error = [] - sshmap.status_clear() + status_clear() # If summarize_failures option is set don't print ssh errors inline if result.setting('summarize_failed') and result.ssh_retcode: return result diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 203bf78..6c8eafc 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -16,11 +16,13 @@ """ #disable deprecated warning messages import warnings +import sshmap.utility.get_parm_val +import sshmap.utility.status_info +import sshmap.utility.status_clear warnings.filterwarnings("ignore") # Python Standard Library imports -import sys import os import getpass import socket @@ -35,6 +37,8 @@ # Imports from other sshmap modules import hostlists + +import sshmap.utility import sshmap.callback # Defaults @@ -125,7 +129,7 @@ def setting(self, key): """ Get a setting from the parm dict or return None if it doesn't exist """ - return get_parm_val(self.parm, key) + return sshmap.utility.get_parm_val(self.parm, key) def ssh_error_message(self): """ Return the ssh_error_message for the error code """ @@ -179,7 +183,7 @@ def setting(self, key): """ Get a setting from the parm dict or return None if it doesn't exist """ - return get_parm_val(self.parm, key) + return sshmap.utility.get_parm_val(self.parm, key) def agent_auth(transport, username): @@ -401,43 +405,6 @@ def run_command(host, command="uname -a", username=None, password=None, return result -# Handy utility functions -def get_parm_val(parm=None, key=None): - """ - Return the value of a key - - >>> get_parm_val(parm={'test':'val'},key='test') - 'val' - >>> get_parm_val(parm={'test':'val'},key='foo') - >>> - """ - if parm and key in parm.keys(): - return parm[key] - else: - return None - - -def status_info(callbacks, text): - """ - Update the display line at the cursor - """ - #print callbacks,text - #return - if isinstance(callbacks, list) and \ - sshmap.callback.status_count in callbacks: - status_clear() - sys.stderr.write(text) - sys.stderr.flush() - - -def status_clear(): - """ - Clear the status line (current line) - """ - sys.stderr.write('\x1b[0G\x1b[0K') - #sys.stderr.flush() - - def init_worker(): """ Set up the signal handler for new worker threads """ signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -470,11 +437,11 @@ def run(host_range, command, username=None, password=None, sudo=False, localhost ok 0 0 {'failures': [], 'total_host_count': 1, 'completed_host_count': 1} """ - status_info(output_callback, 'Looking up hosts') + sshmap.utility.status_info(output_callback, 'Looking up hosts') hosts = hostlists.expand(hostlists.range_split(host_range)) if shuffle: random.shuffle(hosts) - status_clear() + sshmap.utility.status_clear() results = ssh_results() if parms: @@ -508,8 +475,8 @@ def run(host_range, command, username=None, password=None, sudo=False, results.parm['total_host_count'] = len(hosts) results.parm['completed_host_count'] = 0 - status_clear() - status_info(output_callback, 'Spawning processes') + sshmap.utility.status_clear() + sshmap.utility.status_info(output_callback, 'Spawning processes') if jobs > len(hosts): jobs = len(hosts) @@ -539,8 +506,8 @@ def run(host_range, command, username=None, password=None, sudo=False, # Create a process pool and pass the parameters to it - status_clear() - status_info( + sshmap.utility.status_clear() + sshmap.utility.status_info( output_callback, 'Sending %d commands to each process' % chunksize) if sshmap.callback.status_count in output_callback: sshmap.callback.status_count(ssh_result(parm=results.parm)) @@ -574,7 +541,7 @@ def run(host_range, command, username=None, password=None, sudo=False, pool.terminate() if isinstance(output_callback, types.ListType) and \ sshmap.callback.status_count in output_callback: - status_clear() + sshmap.utility.status_clear() return results diff --git a/sshmap/utility.py b/sshmap/utility.py new file mode 100644 index 0000000..ebd3711 --- /dev/null +++ b/sshmap/utility.py @@ -0,0 +1,53 @@ +#Copyright (c) 2012 Yahoo! Inc. All rights reserved. +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +""" +sshmap utility functions +""" +import sys + +__author__ = 'dhubbard' + + +def get_parm_val(parm=None, key=None): + """ + Return the value of a key + + >>> get_parm_val(parm={'test':'val'},key='test') + 'val' + >>> get_parm_val(parm={'test':'val'},key='foo') + >>> + """ + if parm and key in parm.keys(): + return parm[key] + else: + return None + + +def status_info(callbacks, text): + """ + Update the display line at the cursor + """ + #print callbacks,text + #return + if isinstance(callbacks, list) and \ + sshmap.callback.status_count in callbacks: + status_clear() + sys.stderr.write(text) + sys.stderr.flush() + + +def status_clear(): + """ + Clear the status line (current line) + """ + sys.stderr.write('\x1b[0G\x1b[0K') + #sys.stderr.flush() \ No newline at end of file From 0a1e41c8421c32ec8f2a4e914332bdbbea965261 Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Mon, 8 Jul 2013 21:43:33 -0700 Subject: [PATCH 04/52] Fix sshmap shell command --- setup.py | 2 +- sshmap/__init__.py | 0 sshmap/callback.py | 0 sshmap/sshmap | 19 ++++++++++--------- sshmap/utility.py | 0 5 files changed, 11 insertions(+), 10 deletions(-) mode change 100644 => 100755 sshmap/__init__.py mode change 100644 => 100755 sshmap/callback.py mode change 100644 => 100755 sshmap/utility.py diff --git a/setup.py b/setup.py index 0915cef..213a2ba 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.8", + version="0.5.9", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/__init__.py b/sshmap/__init__.py old mode 100644 new mode 100755 diff --git a/sshmap/callback.py b/sshmap/callback.py old mode 100644 new mode 100755 diff --git a/sshmap/sshmap b/sshmap/sshmap index 9f2aae5..a8d1906 100755 --- a/sshmap/sshmap +++ b/sshmap/sshmap @@ -18,7 +18,8 @@ import sys from optparse import OptionParser import getpass -from sshmap import * +import sshmap + if __name__ == "__main__": parser = OptionParser() @@ -76,22 +77,22 @@ if __name__ == "__main__": options.username = getpass.getuser() options.output = True # Create our callback pipeline based on the options passed - callback = [callback_summarize_failures] + callback = [sshmap.callback.summarize_failures] if options.match: - callback.append(callback_filter_match) + callback.append(sshmap.callback.filter_match) if options.output_base64: - callback.append(callback_filter_base64) + callback.append(sshmap.callback.filter_base64) if options.output_json: - callback.append(callback_filter_json) + callback.append(sshmap.callback.filter_json) if options.callback_script: - callback.append(callback_exec_command) + callback.append(sshmap.callback.exec_command) else: if options.aggregate_output: - callback.append(callback_aggregate_output) + callback.append(sshmap.callback.aggregate_output) else: - callback.append(callback_output_prefix_host) + callback.append(sshmap.callback.output_prefix_host) if options.show_status: - callback.append(callback_status_count) + callback.append(sshmap.callback.status_count) # Get the password if the options passed indicate it might be needed if options.sudo: # Prompt for password, we really need to add a password file option diff --git a/sshmap/utility.py b/sshmap/utility.py old mode 100644 new mode 100755 From 72fc2048accb9f1d79700e36316ce5c8cbe00df5 Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Mon, 8 Jul 2013 21:57:49 -0700 Subject: [PATCH 05/52] Fix sshmap shell command --- setup.py | 2 +- sshmap/__init__.py | 3 ++- sshmap/sshmap | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 213a2ba..2630afd 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.9", + version="0.5.10", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/__init__.py b/sshmap/__init__.py index 07978b1..f39190b 100755 --- a/sshmap/__init__.py +++ b/sshmap/__init__.py @@ -10,4 +10,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -from sshmap import * + +from sshmap import run, run_command, ssh_result, ssh_results, fastSSHClient diff --git a/sshmap/sshmap b/sshmap/sshmap index a8d1906..6e1190e 100755 --- a/sshmap/sshmap +++ b/sshmap/sshmap @@ -19,6 +19,7 @@ import sys from optparse import OptionParser import getpass import sshmap +import sshmap.callback if __name__ == "__main__": From 25a9fe384cd3221d0f684185606f098bdcab18bd Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Mon, 8 Jul 2013 22:06:28 -0700 Subject: [PATCH 06/52] Fix sshmap shell command --- setup.py | 2 +- sshmap/sshmap.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 2630afd..e3f419e 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.10", + version="0.5.14", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 6c8eafc..5f29e31 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -16,9 +16,6 @@ """ #disable deprecated warning messages import warnings -import sshmap.utility.get_parm_val -import sshmap.utility.status_info -import sshmap.utility.status_clear warnings.filterwarnings("ignore") From 7fbf355fff8536e2a468c49e5c5cbabff4e763ab Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Tue, 9 Jul 2013 00:42:20 -0700 Subject: [PATCH 07/52] Another attempt at fixing sudo --- setup.py | 2 +- sshmap/__init__.py | 6 ++- sshmap/callback.py | 9 ++-- sshmap/sshmap | 16 +++---- sshmap/sshmap.py | 102 ++++++++++++++++++++++++++------------------- sshmap/utility.py | 3 +- 6 files changed, 79 insertions(+), 59 deletions(-) diff --git a/setup.py b/setup.py index e3f419e..4cc22e1 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.14", + version="0.5.32", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/__init__.py b/sshmap/__init__.py index f39190b..481c85c 100755 --- a/sshmap/__init__.py +++ b/sshmap/__init__.py @@ -11,4 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -from sshmap import run, run_command, ssh_result, ssh_results, fastSSHClient +#from sshmap import run, run_command, ssh_result, ssh_results, fastSSHClient +import sshmap +import callback +import utility +from sshmap import run, run_command \ No newline at end of file diff --git a/sshmap/callback.py b/sshmap/callback.py index 4025eb0..07437e7 100755 --- a/sshmap/callback.py +++ b/sshmap/callback.py @@ -9,8 +9,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -from sshmap.utility import status_clear - __author__ = 'dhubbard' """ sshmap built in callback handlers @@ -23,7 +21,8 @@ import base64 import subprocess -import sshmap +import utility + # Filter callback handlers def flowthrough(result): @@ -59,7 +58,7 @@ def exec_command(result): script = result.setting("callback_script") if not script: return result - status_clear() + utility.status_clear() result_out, result_err = subprocess.Popen( script + " " + result.host, shell=True, @@ -169,7 +168,7 @@ def output_prefix_host(result): """ output = [] error = [] - status_clear() + utility.status_clear() # If summarize_failures option is set don't print ssh errors inline if result.setting('summarize_failed') and result.ssh_retcode: return result diff --git a/sshmap/sshmap b/sshmap/sshmap index 6e1190e..73b39df 100755 --- a/sshmap/sshmap +++ b/sshmap/sshmap @@ -15,15 +15,15 @@ Python based ssh multiplexer optimized for map operations command line utility. """ +import os import sys -from optparse import OptionParser +import optparse import getpass import sshmap -import sshmap.callback - +import hostlists if __name__ == "__main__": - parser = OptionParser() + parser = optparse.OptionParser() #parser.add_option("--html", dest="html", default=False,action="store_true", help="Use HTML for formatting") parser.add_option("--output_json", dest = "output_json", default = False, action = "store_true", help = "Output in JSON format") @@ -110,9 +110,9 @@ if __name__ == "__main__": command = ' '.join(args[1:]) range = args[0] - results = run(args[0], command, username = options.username, password = options.password, sudo = options.sudo, - timeout = options.timeout, script = options.runscript, jobs = options.jobs, sort = options.sort, - shuffle = options.shuffle, output_callback = callback, parms = vars(options)) + results = sshmap.run(args[0], command, username=options.username, password=options.password, sudo=options.sudo, + timeout=options.timeout, script=options.runscript, jobs=options.jobs, sort=options.sort, + shuffle=options.shuffle, output_callback=callback, parms=vars(options)) if options.aggregate_output: aggregate_hosts = results.setting('aggregate_hosts') collapsed_output = results.setting('collapsed_output') @@ -123,7 +123,7 @@ if __name__ == "__main__": print ','.join(hostlists.compress(aggregate_hosts[md5])) print "-" * (int(columns)-2) stdout, stderr = collapsed_output[md5] - if len(stdout): print '\n'.join(stdout) + if len(stdout): print ''.join(stdout) if len(stderr): print >> sys.stderr, '\n'.join(stderr) if options.summarize_failed and 'failures' in results.parm.keys() and len(results.parm['failures']): print 'SSH Failed to: %s' % hostlists.compress(results.parm['failures']) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 5f29e31..b0cc267 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -34,9 +34,8 @@ # Imports from other sshmap modules import hostlists - -import sshmap.utility -import sshmap.callback +import utility +import callback # Defaults JOB_MAX = 100 @@ -86,7 +85,13 @@ def wrapper(func): + """ + Simple timeout wrapper for multiprocessing + """ def wrap(self, timeout=None): + """ + The wrapper method + """ return func(self, timeout=timeout if timeout is not None else 1e100) return wrap @@ -95,7 +100,7 @@ def wrap(self, timeout=None): IMapIterator.next = wrapper(IMapIterator.next) -class ssh_result: +class ssh_result(object): """ ssh_result class, that holds the output from the ssh_call. This is passed to all the callback functions. @@ -126,7 +131,7 @@ def setting(self, key): """ Get a setting from the parm dict or return None if it doesn't exist """ - return sshmap.utility.get_parm_val(self.parm, key) + return utility.get_parm_val(self.parm, key) def ssh_error_message(self): """ Return the ssh_error_message for the error code """ @@ -180,7 +185,7 @@ def setting(self, key): """ Get a setting from the parm dict or return None if it doesn't exist """ - return sshmap.utility.get_parm_val(self.parm, key) + return utility.get_parm_val(self.parm, key) def agent_auth(transport, username): @@ -324,7 +329,7 @@ def run_command(host, command="uname -a", username=None, password=None, if sudo: stdin, stdout, stderr, chan = client.exec_command( 'sudo -k -S %s' % command, - timeout=timeout, bufsize=bufsize, pty=True + timeout=timeout, bufsize=bufsize, pty=False ) if not chan: result.ssh_retcode = RUN_FAIL_CONNECT @@ -345,29 +350,33 @@ def run_command(host, command="uname -a", username=None, password=None, # Send the password stdin.write(password + '\r') stdin.flush() - - # Remove the password prompt and password from the output - prompt = _term_readline(stdout) - seen_password = False - seen_password_prompt = False - #print 'READ:',prompt - while 'assword:' in prompt or password in prompt or \ - 'try again' in prompt or len(prompt.strip()) == 0: - if 'try again' in prompt: - result.ssh_retcode = RUN_FAIL_BADPASSWORD - return result - prompt_new = _term_readline(stdout) - if 'assword:' in prompt: - seen_password_prompt = True - if password in prompt: - seen_password = True - if seen_password_prompt and seen_password: - break - prompt = prompt_new + + if False: + # Remove the password prompt and password from the output + # should only be needed if using a pty + prompt = _term_readline(stdout) + seen_password = False + seen_password_prompt = False + #print 'READ:',prompt + while 'assword:' in prompt or False or password in prompt or \ + 'try again' in prompt or len(prompt.strip()) == 0: + if 'try again' in prompt: + result.ssh_retcode = RUN_FAIL_BADPASSWORD + return result + prompt_new = _term_readline(stdout) + if 'assword:' in prompt: + seen_password_prompt = True + if password in prompt: + seen_password = True + if seen_password_prompt or seen_password: + break + prompt = prompt_new except socket.timeout: result.err = ['Timeout during sudo connect, likely bad password'] result.ssh_retcode = RUN_FAIL_TIMEOUT return result + result.out = [] + result.err = [] if script: # Pass the script over stdin and close the channel so the receving end # gets an EOF process it as a django template with the arguments passed @@ -388,10 +397,17 @@ def run_command(host, command="uname -a", username=None, password=None, stdin.write(open(script, 'r').read()) stdin.flush() stdin.channel.shutdown_write() + if sudo: + prompt = _term_readline(stderr) + if prompt and 'assword' not in prompt and password not in prompt: + result.err = [prompt] + #prompt = _term_readline(stderr) + try: # Read the output from stdout,stderr and close the connection - result.out = stdout.readlines() - result.err = stderr.readlines() + result.out = result.out + stdout.readlines() + result.err = result.err + stderr.readlines() + #print result.err result.retcode = chan.recv_exit_status() if close_client: client.close() @@ -409,7 +425,7 @@ def init_worker(): def run(host_range, command, username=None, password=None, sudo=False, script=None, timeout=None, sort=False, bufsize=-1, cwd='/tmp', - jobs=None, output_callback=sshmap.callback.summarize_failures, + jobs=None, output_callback=callback.summarize_failures, parms=None, shuffle=False, chunksize=None): """ Run a command on a hostlists host_range of hosts @@ -434,11 +450,11 @@ def run(host_range, command, username=None, password=None, sudo=False, localhost ok 0 0 {'failures': [], 'total_host_count': 1, 'completed_host_count': 1} """ - sshmap.utility.status_info(output_callback, 'Looking up hosts') + utility.status_info(output_callback, 'Looking up hosts') hosts = hostlists.expand(hostlists.range_split(host_range)) if shuffle: random.shuffle(hosts) - sshmap.utility.status_clear() + utility.status_clear() results = ssh_results() if parms: @@ -472,8 +488,8 @@ def run(host_range, command, username=None, password=None, sudo=False, results.parm['total_host_count'] = len(hosts) results.parm['completed_host_count'] = 0 - sshmap.utility.status_clear() - sshmap.utility.status_info(output_callback, 'Spawning processes') + utility.status_clear() + utility.status_info(output_callback, 'Spawning processes') if jobs > len(hosts): jobs = len(hosts) @@ -498,16 +514,16 @@ def run(host_range, command, username=None, password=None, sudo=False, map_command = pool.imap_unordered if isinstance(output_callback, types.ListType) and \ - sshmap.callback.status_count in output_callback: - sshmap.callback.status_count(ssh_result(parm=results.parm)) + callback.status_count in output_callback: + callback.status_count(ssh_result(parm=results.parm)) # Create a process pool and pass the parameters to it - sshmap.utility.status_clear() - sshmap.utility.status_info( + utility.status_clear() + utility.status_info( output_callback, 'Sending %d commands to each process' % chunksize) - if sshmap.callback.status_count in output_callback: - sshmap.callback.status_count(ssh_result(parm=results.parm)) + if callback.status_count in output_callback: + callback.status_count(ssh_result(parm=results.parm)) try: for result in map_command( @@ -522,8 +538,8 @@ def run(host_range, command, username=None, password=None, sudo=False, results.parm['completed_host_count'] += 1 result.parm = results.parm if isinstance(output_callback, types.ListType): - for callback in output_callback: - result = callback(result) + for cb in output_callback: + result = cb(result) else: result = output_callback(result) results.parm = result.parm @@ -537,8 +553,8 @@ def run(host_range, command, username=None, password=None, sudo=False, # pass pool.terminate() if isinstance(output_callback, types.ListType) and \ - sshmap.callback.status_count in output_callback: - sshmap.utility.status_clear() + callback.status_count in output_callback: + utility.status_clear() return results diff --git a/sshmap/utility.py b/sshmap/utility.py index ebd3711..5073544 100755 --- a/sshmap/utility.py +++ b/sshmap/utility.py @@ -13,6 +13,7 @@ sshmap utility functions """ import sys +import callback __author__ = 'dhubbard' @@ -39,7 +40,7 @@ def status_info(callbacks, text): #print callbacks,text #return if isinstance(callbacks, list) and \ - sshmap.callback.status_count in callbacks: + callback.status_count in callbacks: status_clear() sys.stderr.write(text) sys.stderr.flush() From b3016668b6d832dbbb7bc02f6dc4285249992d46 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 11:31:30 -0700 Subject: [PATCH 08/52] Remove extra passwords and prompts --- setup.py | 2 +- sshmap/sshmap.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4cc22e1..c624c33 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.32", + version="0.5.33", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index b0cc267..76a4ee7 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -401,6 +401,8 @@ def run_command(host, command="uname -a", username=None, password=None, prompt = _term_readline(stderr) if prompt and 'assword' not in prompt and password not in prompt: result.err = [prompt] + if prompt and 'assword' not in prompt and password not in prompt: + result.err.append(prompt) #prompt = _term_readline(stderr) try: From 5ed4c59a70be883b364148da60609f606726ba10 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 11:34:37 -0700 Subject: [PATCH 09/52] Remove apssword prompt when using sudo --- setup.py | 2 +- sshmap/sshmap.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c624c33..1c714ba 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.33", + version="0.5.34", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 76a4ee7..0008020 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -400,7 +400,7 @@ def run_command(host, command="uname -a", username=None, password=None, if sudo: prompt = _term_readline(stderr) if prompt and 'assword' not in prompt and password not in prompt: - result.err = [prompt] + result.err.append(prompt) if prompt and 'assword' not in prompt and password not in prompt: result.err.append(prompt) #prompt = _term_readline(stderr) From 09f3b6bc99bb980a8613c5996fb6b021881fa66f Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 11:44:23 -0700 Subject: [PATCH 10/52] Add additional prompt check --- sshmap/sshmap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 0008020..366c88a 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -401,6 +401,7 @@ def run_command(host, command="uname -a", username=None, password=None, prompt = _term_readline(stderr) if prompt and 'assword' not in prompt and password not in prompt: result.err.append(prompt) + prompt = _term_readline(stderr) if prompt and 'assword' not in prompt and password not in prompt: result.err.append(prompt) #prompt = _term_readline(stderr) From 6e61f8bb0d2b3111886b80bc7239e5a84dfcaf0d Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 11:53:15 -0700 Subject: [PATCH 11/52] Add debug prints --- setup.py | 2 +- sshmap/sshmap.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1c714ba..0ce53f8 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.34", + version="0.5.35", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 366c88a..f8041a7 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -400,9 +400,11 @@ def run_command(host, command="uname -a", username=None, password=None, if sudo: prompt = _term_readline(stderr) if prompt and 'assword' not in prompt and password not in prompt: + print 'Removing prompt', prompt.strip() result.err.append(prompt) prompt = _term_readline(stderr) if prompt and 'assword' not in prompt and password not in prompt: + print 'Removing prompt', prompt.strip() result.err.append(prompt) #prompt = _term_readline(stderr) From 9fe02aa70cb02a4ea3c6fc923e12584610e78017 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 12:22:35 -0700 Subject: [PATCH 12/52] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0ce53f8..4890909 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.35", + version="0.5.36", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", From ef9136f23f89108f7e30bbac92402d7864e237a8 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 12:34:01 -0700 Subject: [PATCH 13/52] Try and fix sudo hanging again --- setup.py | 2 +- sshmap/sshmap.py | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 4890909..8208108 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.36", + version="0.5.37", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index f8041a7..6d1dacc 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -397,17 +397,6 @@ def run_command(host, command="uname -a", username=None, password=None, stdin.write(open(script, 'r').read()) stdin.flush() stdin.channel.shutdown_write() - if sudo: - prompt = _term_readline(stderr) - if prompt and 'assword' not in prompt and password not in prompt: - print 'Removing prompt', prompt.strip() - result.err.append(prompt) - prompt = _term_readline(stderr) - if prompt and 'assword' not in prompt and password not in prompt: - print 'Removing prompt', prompt.strip() - result.err.append(prompt) - #prompt = _term_readline(stderr) - try: # Read the output from stdout,stderr and close the connection result.out = result.out + stdout.readlines() From 96cdf5181795678137b09d03fab831f67f8eddf7 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 12:45:39 -0700 Subject: [PATCH 14/52] Undo more changes --- sshmap/sshmap.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 6d1dacc..38ab800 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -375,8 +375,6 @@ def run_command(host, command="uname -a", username=None, password=None, result.err = ['Timeout during sudo connect, likely bad password'] result.ssh_retcode = RUN_FAIL_TIMEOUT return result - result.out = [] - result.err = [] if script: # Pass the script over stdin and close the channel so the receving end # gets an EOF process it as a django template with the arguments passed @@ -399,8 +397,8 @@ def run_command(host, command="uname -a", username=None, password=None, stdin.channel.shutdown_write() try: # Read the output from stdout,stderr and close the connection - result.out = result.out + stdout.readlines() - result.err = result.err + stderr.readlines() + result.out = stdout.readlines() + result.err = stderr.readlines() #print result.err result.retcode = chan.recv_exit_status() if close_client: From 713089acb27b94e9662bd4a4ce4c057b572ed052 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 12:46:04 -0700 Subject: [PATCH 15/52] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8208108..3d6efd7 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.37", + version="0.5.38", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", From 0d2da0514a2c63a5e1b1aa3056ec7b86806cc59b Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 13:30:27 -0700 Subject: [PATCH 16/52] Add back the code to skip the sudo password and prompts --- setup.py | 2 +- sshmap/sshmap.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3d6efd7..697231b 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.38", + version="0.5.39", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 38ab800..4e08cd8 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -399,6 +399,22 @@ def run_command(host, command="uname -a", username=None, password=None, # Read the output from stdout,stderr and close the connection result.out = stdout.readlines() result.err = stderr.readlines() + if sudo: + # Remove any passwords or prompts from the start of the stderr + # output + err = [] + check_prompt = True + skip = False + for el in result.err: + if check_prompt: + if password in el or 'assword:' in el: + skip = True + else: + check_prompt = False + if not skip: + err.append(el) + skip = False + #print result.err result.retcode = chan.recv_exit_status() if close_client: From e77e25ce3fff62e99f04247a4aa26b6109b37f02 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 9 Jul 2013 13:33:01 -0700 Subject: [PATCH 17/52] Forgot to set it back to the result.err --- setup.py | 2 +- sshmap/sshmap.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 697231b..ca339e5 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.39", + version="0.5.40", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 4e08cd8..1fd0d26 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -414,6 +414,7 @@ def run_command(host, command="uname -a", username=None, password=None, if not skip: err.append(el) skip = False + result.err = err #print result.err result.retcode = chan.recv_exit_status() From 84c37fff188f3e019f0908f11d96848163f23323 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 12:10:25 -0700 Subject: [PATCH 18/52] Clean up old python2 format print in dump --- sshmap/sshmap.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 1fd0d26..975170d 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -21,6 +21,7 @@ # Python Standard Library imports import os +import sys import getpass import socket import types @@ -87,6 +88,7 @@ def wrapper(func): """ Simple timeout wrapper for multiprocessing + :param func: """ def wrap(self, timeout=None): """ @@ -130,6 +132,7 @@ def err_string(self): def setting(self, key): """ Get a setting from the parm dict or return None if it doesn't exist + :param key: """ return utility.get_parm_val(self.parm, key) @@ -138,14 +141,19 @@ def ssh_error_message(self): return RUN_CODES[self.ssh_retcode] def dump(self, return_parm=True, return_retcode=True): - """ Print all our public values """ - print self.host, self.out_string().replace('\n', ''), self.err_string().replace('\n', ''), + """ Print all our public values + :param return_parm: + :param return_retcode: + """ + sys.stdout.write(self.host+' ') + sys.stdout.write(self.out_string().replace('\n', '')+' ') + sys.stderr.write(self.err_string().replace('\n', '')+' ') if return_retcode: - print self.retcode, + sys.stdout.write(self.retcode+' ') if return_parm: - print self.ssh_retcode, self.parm + sys.stdout.write(self.ssh_retcode+' '+self.parm) else: - print + sys.stdout.write('\n') def print_output(self): """ Print output from the commands """ From 8e4fdc2d32d2db9b918ccdeee596a87a416b7392 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 13:51:47 -0700 Subject: [PATCH 19/52] Moved default global variables out of the main script file --- sshmap/defaults.py | 59 ++++++++++++++++++++++ sshmap/sshmap.py | 120 ++++++++++++++++----------------------------- 2 files changed, 102 insertions(+), 77 deletions(-) create mode 100644 sshmap/defaults.py diff --git a/sshmap/defaults.py b/sshmap/defaults.py new file mode 100644 index 0000000..191b09a --- /dev/null +++ b/sshmap/defaults.py @@ -0,0 +1,59 @@ +#Copyright (c) 2012 Yahoo! Inc. All rights reserved. +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +""" +Default Values for sshmap +""" +__author__ = 'dhubbard' +import os + +# Defaults +JOB_MAX = 100 +# noinspection PyBroadException +try: + for line in open('/proc/%d/limits' % os.getpid(), 'r').readlines(): + if line.startswith('Max processes'): + JOB_MAX = int(line.strip().split()[2]) / 4 +except: + pass + +# Return code values +RUN_OK = 0 +RUN_FAIL_AUTH = 1 +RUN_FAIL_TIMEOUT = 2 +RUN_FAIL_CONNECT = 3 +RUN_FAIL_SSH = 4 +RUN_SUDO_PROMPT = 5 +RUN_FAIL_UNKNOWN = 6 +RUN_FAIL_NOPASSWORD = 7 +RUN_FAIL_BADPASSWORD = 8 + +# Text return codes +RUN_CODES = ['Ok', 'Authentication Error', 'Timeout', 'SSH Connection Failed', + 'SSH Failure', + 'Sudo did not send a password prompt', 'Connection refused', + 'Sudo password required', + 'Invalid sudo password'] + +# Configuration file field descriptions +conf_desc = { + "username": "IRC Server username", + "password": "IRC Server password", + "channel": "sshmap", +} + +# Configuration file defaults +conf_defaults = { + "address": "chat.freenode.net", + "port": "6667", + "use_ssl": False, +} diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 975170d..0da9475 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -14,9 +14,11 @@ """ Python based ssh multiplexer optimized for map operations """ -#disable deprecated warning messages -import warnings +# Pull in the python3 print function for python3 compatibility +from __future__ import print_function +#disable deprecated warning messages that occur during some of the imports +import warnings warnings.filterwarnings("ignore") # Python Standard Library imports @@ -37,48 +39,8 @@ import hostlists import utility import callback +import defaults -# Defaults -JOB_MAX = 100 -# noinspection PyBroadException -try: - for line in open('/proc/%d/limits' % os.getpid(), 'r').readlines(): - if line.startswith('Max processes'): - JOB_MAX = int(line.strip().split()[2]) / 4 -except: - pass - -# Return code values -RUN_OK = 0 -RUN_FAIL_AUTH = 1 -RUN_FAIL_TIMEOUT = 2 -RUN_FAIL_CONNECT = 3 -RUN_FAIL_SSH = 4 -RUN_SUDO_PROMPT = 5 -RUN_FAIL_UNKNOWN = 6 -RUN_FAIL_NOPASSWORD = 7 -RUN_FAIL_BADPASSWORD = 8 - -# Text return codes -RUN_CODES = ['Ok', 'Authentication Error', 'Timeout', 'SSH Connection Failed', - 'SSH Failure', - 'Sudo did not send a password prompt', 'Connection refused', - 'Sudo password required', - 'Invalid sudo password'] - -# Configuration file field descriptions -conf_desc = { - "username": "IRC Server username", - "password": "IRC Server password", - "channel": "sshmap", -} - -# Configuration file defaults -conf_defaults = { - "address": "chat.freenode.net", - "port": "6667", - "use_ssl": False, -} # Fix to make ctrl-c correctly terminate child processes # spawned by the multiprocessing module @@ -93,6 +55,7 @@ def wrapper(func): def wrap(self, timeout=None): """ The wrapper method + :param timeout: """ return func(self, timeout=timeout if timeout is not None else 1e100) @@ -138,7 +101,7 @@ def setting(self, key): def ssh_error_message(self): """ Return the ssh_error_message for the error code """ - return RUN_CODES[self.ssh_retcode] + return defaults.RUN_CODES[self.ssh_retcode] def dump(self, return_parm=True, return_retcode=True): """ Print all our public values @@ -176,7 +139,7 @@ def dump(self): """ Dump all the result objects """ for item in self.__iter__(): item.dump(return_parm=False, return_retcode=False) - print self.parm + print(self.parm) def print_output(self, summarize_failures=False): """ Print all the objects """ @@ -192,15 +155,19 @@ def print_output(self, summarize_failures=False): def setting(self, key): """ Get a setting from the parm dict or return None if it doesn't exist + :param key: """ return utility.get_parm_val(self.parm, key) + def agent_auth(transport, username): """ Attempt to authenticate to the given transport using any of the private keys available from an SSH agent or from a local private RSA key file (assumes no pass phrase). + :param transport: + :param username: """ agent = ssh.Agent() @@ -252,14 +219,14 @@ def _term_readline(handle): if char in ['\r', '\n']: return buf char = handle.read(1) - except Exception, message: - print Exception, message + except Exception as message: + print('%s %s' % (Exception, message)) return buf def run_command(host, command="uname -a", username=None, password=None, sudo=False, script=None, timeout=None, parms=None, client=None, - bufsize=-1, cwd='/tmp', logging=False): + bufsize=-1, logging=False): """ Run a command or script on a remote node via ssh :param host: @@ -277,7 +244,8 @@ def run_command(host, command="uname -a", username=None, password=None, """ # Guess any parameters not passed that can be if isinstance(host, types.TupleType): - host, command, username, password, sudo, script, timeout, parms, client = host + host, command, username, password, sudo, script, timeout, parms, \ + client = host if timeout == 0: timeout = None if not username: @@ -306,7 +274,7 @@ def run_command(host, command="uname -a", username=None, password=None, client = fastSSHClient() except: result.err = ['Error creating client'] - result.ssh_retcode = RUN_FAIL_UNKNOWN + result.ssh_retcode = defaults.RUN_FAIL_UNKNOWN return result client.set_missing_host_key_policy(ssh.AutoAddPolicy()) # load_system_host_keys slows things way down @@ -317,19 +285,19 @@ def run_command(host, command="uname -a", username=None, password=None, client.connect(host, username=username, password=password, timeout=timeout) except ssh.AuthenticationException: - result.ssh_retcode = RUN_FAIL_AUTH + result.ssh_retcode = defaults.RUN_FAIL_AUTH return result except ssh.SSHException: - result.ssh_retcode = RUN_FAIL_CONNECT + result.ssh_retcode = defaults.RUN_FAIL_CONNECT return result except AttributeError: - result.ssh_retcode = RUN_FAIL_SSH + result.ssh_retcode = defaults.RUN_FAIL_SSH return result except socket.error: - result.ssh_retcode = RUN_FAIL_CONNECT + result.ssh_retcode = defaults.RUN_FAIL_CONNECT return result - except Exception, message: - result.ssh_retcode = RUN_FAIL_UNKNOWN + except Exception as message: + result.ssh_retcode = defaults.RUN_FAIL_UNKNOWN return result try: # We have to force a sudo -k first or we can't reliably know we'll be @@ -340,18 +308,18 @@ def run_command(host, command="uname -a", username=None, password=None, timeout=timeout, bufsize=bufsize, pty=False ) if not chan: - result.ssh_retcode = RUN_FAIL_CONNECT + result.ssh_retcode = defaults.RUN_FAIL_CONNECT return result else: stdin, stdout, stderr, chan = client.exec_command( command, timeout=timeout, bufsize=bufsize) if not chan: - result.ssh_retcode = RUN_FAIL_CONNECT + result.ssh_retcode = defaults.RUN_FAIL_CONNECT result.err = ["WTF, this shouldn't happen\n"] return result - except ssh.SSHException, ssh.transport.SSHException: - result.ssh_retcode = RUN_FAIL_SSH + except (ssh.SSHException, ssh.transport.SSHException): + result.ssh_retcode = defaults.RUN_FAIL_SSH return result if sudo: try: @@ -369,7 +337,7 @@ def run_command(host, command="uname -a", username=None, password=None, while 'assword:' in prompt or False or password in prompt or \ 'try again' in prompt or len(prompt.strip()) == 0: if 'try again' in prompt: - result.ssh_retcode = RUN_FAIL_BADPASSWORD + result.ssh_retcode = defaults.RUN_FAIL_BADPASSWORD return result prompt_new = _term_readline(stdout) if 'assword:' in prompt: @@ -381,7 +349,7 @@ def run_command(host, command="uname -a", username=None, password=None, prompt = prompt_new except socket.timeout: result.err = ['Timeout during sudo connect, likely bad password'] - result.ssh_retcode = RUN_FAIL_TIMEOUT + result.ssh_retcode = defaults.RUN_FAIL_TIMEOUT return result if script: # Pass the script over stdin and close the channel so the receving end @@ -395,9 +363,9 @@ def run_command(host, command="uname -a", username=None, password=None, django.conf.settings.configure() template = open(script, 'r').read() if script_parameters: - c = django.template.Context({ 'argv': script_parameters }) + c = django.template.Context({'argv': script_parameters}) else: - c = django.template.Context({ }) + c = django.template.Context({}) stdin.write(django.template.Template(template).render(c)) except Exception as e: stdin.write(open(script, 'r').read()) @@ -429,9 +397,9 @@ def run_command(host, command="uname -a", username=None, password=None, if close_client: client.close() except socket.timeout: - result.ssh_retcode = RUN_FAIL_TIMEOUT + result.ssh_retcode = defaults.RUN_FAIL_TIMEOUT return result - result.ssh_retcode = RUN_OK + result.ssh_retcode = defaults.RUN_OK return result @@ -441,7 +409,7 @@ def init_worker(): def run(host_range, command, username=None, password=None, sudo=False, - script=None, timeout=None, sort=False, bufsize=-1, cwd='/tmp', + script=None, timeout=None, sort=False, jobs=None, output_callback=callback.summarize_failures, parms=None, shuffle=False, chunksize=None): """ @@ -454,8 +422,6 @@ def run(host_range, command, username=None, password=None, sudo=False, :param script: :param timeout: :param sort: - :param bufsize: - :param cwd: :param jobs: :param output_callback: :param parms: @@ -477,13 +443,13 @@ def run(host_range, command, username=None, password=None, sudo=False, if parms: results.parm = parms else: - results.parm = { } + results.parm = {} if sudo and not password: for host in hosts: - result=ssh_result() - result.err='Sudo password required' - result.retcode = RUN_FAIL_NOPASSWORD + result = ssh_result() + result.err = 'Sudo password required' + result.retcode = defaults.RUN_FAIL_NOPASSWORD results.append(result) results.parm['total_host_count'] = len(hosts) results.parm['completed_host_count'] = 0 @@ -492,8 +458,8 @@ def run(host_range, command, username=None, password=None, sudo=False, if jobs < 1: jobs = 1 - if jobs > JOB_MAX: - jobs = JOB_MAX + if jobs > defaults.JOB_MAX: + jobs = defaults.JOB_MAX # Set up our ssh client #status_info(output_callback,'Setting up the SSH client') @@ -513,7 +479,6 @@ def run(host_range, command, username=None, password=None, sudo=False, pool = multiprocessing.Pool(processes=jobs, initializer=init_worker) if not chunksize: - chunksize = 1 if jobs >= len(hosts): chunksize = 1 else: @@ -551,7 +516,8 @@ def run(host_range, command, username=None, password=None, sudo=False, results.parm, client ) for host in hosts ], - chunksize): + chunksize + ): results.parm['completed_host_count'] += 1 result.parm = results.parm if isinstance(output_callback, types.ListType): From 283b378ff43b680868e4df8663c7157a36d9518b Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 13:57:32 -0700 Subject: [PATCH 20/52] PEP8 cleanups --- sshmap/sshmap.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 0da9475..44b38dc 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -142,7 +142,9 @@ def dump(self): print(self.parm) def print_output(self, summarize_failures=False): - """ Print all the objects """ + """ Print all the objects + :param summarize_failures: + """ for item in self.__iter__(): item.print_output() if summarize_failures: @@ -160,7 +162,6 @@ def setting(self, key): return utility.get_parm_val(self.parm, key) - def agent_auth(transport, username): """ Attempt to authenticate to the given transport using any of the private @@ -239,13 +240,12 @@ def run_command(host, command="uname -a", username=None, password=None, :param parms: :param client: :param bufsize: - :param cwd: :param logging: """ # Guess any parameters not passed that can be if isinstance(host, types.TupleType): host, command, username, password, sudo, script, timeout, parms, \ - client = host + client = host if timeout == 0: timeout = None if not username: @@ -253,13 +253,12 @@ def run_command(host, command="uname -a", username=None, password=None, if bufsize == -1 and script: bufsize = os.path.getsize(script) + 1024 + script_parameters = None if script: temp = command.split() if len(temp) > 1: command = temp[0] script_parameters = temp - else: - script_parameters = None # Get a result object to put our output in result = ssh_result(host=host, parm=parms) @@ -297,6 +296,7 @@ def run_command(host, command="uname -a", username=None, password=None, result.ssh_retcode = defaults.RUN_FAIL_CONNECT return result except Exception as message: + logging.debug('Got unknown exception %s', message) result.ssh_retcode = defaults.RUN_FAIL_UNKNOWN return result try: @@ -448,6 +448,7 @@ def run(host_range, command, username=None, password=None, sudo=False, if sudo and not password: for host in hosts: result = ssh_result() + result.host = host result.err = 'Sudo password required' result.retcode = defaults.RUN_FAIL_NOPASSWORD results.append(result) From 932fc7c04fdab6eb0ad0dc0d7f14e205f77a9a54 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:34:29 -0700 Subject: [PATCH 21/52] Add unittest --- tests/test.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/test.py diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..f19210a --- /dev/null +++ b/tests/test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +#Copyright (c) 2012 Yahoo! Inc. All rights reserved. +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +__author__ = 'dhubbard' +import sshmap +import os +import unittest + +class TestSSH(unittest.TestCase): + + def set_up(self): + pass + + def test_shell_command_as_user(self): + # Run a ssh command to localhost and verify it works + result = os.popen('sshmap/sshmap localhost echo hello').read().strip() + self.assertEqual('localhost: hello', result) + + #def test_shell_command_with_sudo(self): + # Run a ssh command to localhost and verify it works + #result = os.popen('sshmap/sshmap --sudo localhost echo hello').read().strip() + #self.assertEqual('localhost: hello', result) + + def test_shell_script_as_user(self): + # Run a ssh command to localhost and verify it works + open('testscript.test').write('#!/bin/bash\necho hello\n') + result = os.popen('sshmap/sshmap localhost --runscript testscript.test').read().strip() + self.assertEqual('localhost: hello', result) + os.remove('testscript.test') + + def test_shell_script_as_root(self): + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 95900ae06333c5ff247bdbf51b1bff1df3cc3a3a Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:36:38 -0700 Subject: [PATCH 22/52] Remove incomplete test --- tests/test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test.py b/tests/test.py index f19210a..ed43e86 100644 --- a/tests/test.py +++ b/tests/test.py @@ -17,7 +17,9 @@ import unittest class TestSSH(unittest.TestCase): - + """ + sshmap unit tests + """ def set_up(self): pass @@ -38,7 +40,6 @@ def test_shell_script_as_user(self): self.assertEqual('localhost: hello', result) os.remove('testscript.test') - def test_shell_script_as_root(self): if __name__ == '__main__': unittest.main() \ No newline at end of file From 750610ab6e4c8d4888eded0ccc546f4b25538e29 Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Wed, 10 Jul 2013 22:37:30 +0000 Subject: [PATCH 23/52] Fix permissions --- tests/test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/test.py diff --git a/tests/test.py b/tests/test.py old mode 100644 new mode 100755 From 41d56558d9b3d78e79daee264fc44a9ec532f00e Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:40:07 -0700 Subject: [PATCH 24/52] Set file mode to write --- tests/test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test.py b/tests/test.py index ed43e86..15561b9 100755 --- a/tests/test.py +++ b/tests/test.py @@ -28,14 +28,9 @@ def test_shell_command_as_user(self): result = os.popen('sshmap/sshmap localhost echo hello').read().strip() self.assertEqual('localhost: hello', result) - #def test_shell_command_with_sudo(self): - # Run a ssh command to localhost and verify it works - #result = os.popen('sshmap/sshmap --sudo localhost echo hello').read().strip() - #self.assertEqual('localhost: hello', result) - def test_shell_script_as_user(self): # Run a ssh command to localhost and verify it works - open('testscript.test').write('#!/bin/bash\necho hello\n') + open('testscript.test', 'w').write('#!/bin/bash\necho hello\n') result = os.popen('sshmap/sshmap localhost --runscript testscript.test').read().strip() self.assertEqual('localhost: hello', result) os.remove('testscript.test') From c4af38680c996a31ce494537f9e989406bc6c7ac Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:45:00 -0700 Subject: [PATCH 25/52] Add sudo test --- tests/test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test.py b/tests/test.py index 15561b9..eceb585 100755 --- a/tests/test.py +++ b/tests/test.py @@ -28,6 +28,12 @@ def test_shell_command_as_user(self): result = os.popen('sshmap/sshmap localhost echo hello').read().strip() self.assertEqual('localhost: hello', result) + def test_shell_command_sudo(self): + # Run a ssh command to localhost and verify it works + result = os.popen('sshmap/sshmap localhost --sudo id').read().split('\n')[-1].strip() + self.assertEqual('localhost: hello', result) + self.assertIn(result, 'localhost: uid=0(root) gid=0(root) groups=0(root)') + def test_shell_script_as_user(self): # Run a ssh command to localhost and verify it works open('testscript.test', 'w').write('#!/bin/bash\necho hello\n') From 0ed6cace48f0e046135b57be6664891170bb61b6 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:45:58 -0700 Subject: [PATCH 26/52] Remove wrong assert --- tests/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index eceb585..45b4ac4 100755 --- a/tests/test.py +++ b/tests/test.py @@ -31,7 +31,6 @@ def test_shell_command_as_user(self): def test_shell_command_sudo(self): # Run a ssh command to localhost and verify it works result = os.popen('sshmap/sshmap localhost --sudo id').read().split('\n')[-1].strip() - self.assertEqual('localhost: hello', result) self.assertIn(result, 'localhost: uid=0(root) gid=0(root) groups=0(root)') def test_shell_script_as_user(self): From 19abe17b0252eded16f01f53123aab892be65a98 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:47:34 -0700 Subject: [PATCH 27/52] Fix assert statement --- tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 45b4ac4..4f9891c 100755 --- a/tests/test.py +++ b/tests/test.py @@ -31,7 +31,7 @@ def test_shell_command_as_user(self): def test_shell_command_sudo(self): # Run a ssh command to localhost and verify it works result = os.popen('sshmap/sshmap localhost --sudo id').read().split('\n')[-1].strip() - self.assertIn(result, 'localhost: uid=0(root) gid=0(root) groups=0(root)') + self.assert_('localhost: uid=0(root) gid=0(root) groups=0(root)' in result) def test_shell_script_as_user(self): # Run a ssh command to localhost and verify it works From b465f23511b2f66450bf31f831b7547676ba0c30 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:50:14 -0700 Subject: [PATCH 28/52] Fix command execution --- tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 4f9891c..f4191c2 100755 --- a/tests/test.py +++ b/tests/test.py @@ -30,7 +30,7 @@ def test_shell_command_as_user(self): def test_shell_command_sudo(self): # Run a ssh command to localhost and verify it works - result = os.popen('sshmap/sshmap localhost --sudo id').read().split('\n')[-1].strip() + result = os.popen('sshmap/sshmap localhost --sudo id').read().strip() self.assert_('localhost: uid=0(root) gid=0(root) groups=0(root)' in result) def test_shell_script_as_user(self): From 597581bad6ff7ea15b419dcecc244cbed7a7f9b2 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:52:21 -0700 Subject: [PATCH 29/52] Add sudo script test --- tests/test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test.py b/tests/test.py index f4191c2..8523ae7 100755 --- a/tests/test.py +++ b/tests/test.py @@ -40,6 +40,13 @@ def test_shell_script_as_user(self): self.assertEqual('localhost: hello', result) os.remove('testscript.test') + def test_shell_script_sudo(self): + # Run a ssh command to localhost and verify it works + open('testscript.test', 'w').write('#!/bin/bash\nid\n') + result = os.popen('sshmap/sshmap localhost --runscript testscript.test --sudo').read().strip() + self.assert_('localhost: uid=0(root) gid=0(root) groups=0(root)' in result) + os.remove('testscript.test') + if __name__ == '__main__': unittest.main() \ No newline at end of file From 6269b356d30f18f75024f85191cfee65a2a1cff4 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 10 Jul 2013 15:55:27 -0700 Subject: [PATCH 30/52] PEP8 cleanups --- tests/test.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/test.py b/tests/test.py index 8523ae7..b00d850 100755 --- a/tests/test.py +++ b/tests/test.py @@ -11,11 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. +""" +Unit tests of sshmap +""" __author__ = 'dhubbard' import sshmap import os import unittest + class TestSSH(unittest.TestCase): """ sshmap unit tests @@ -24,29 +28,35 @@ def set_up(self): pass def test_shell_command_as_user(self): - # Run a ssh command to localhost and verify it works + """Run a ssh command to localhost and verify it works """ result = os.popen('sshmap/sshmap localhost echo hello').read().strip() self.assertEqual('localhost: hello', result) def test_shell_command_sudo(self): - # Run a ssh command to localhost and verify it works + """Run a ssh command to localhost using sudo and verify it works""" result = os.popen('sshmap/sshmap localhost --sudo id').read().strip() - self.assert_('localhost: uid=0(root) gid=0(root) groups=0(root)' in result) + self.assert_( + 'localhost: uid=0(root) gid=0(root) groups=0(root)' in result) def test_shell_script_as_user(self): # Run a ssh command to localhost and verify it works open('testscript.test', 'w').write('#!/bin/bash\necho hello\n') - result = os.popen('sshmap/sshmap localhost --runscript testscript.test').read().strip() + result = os.popen( + 'sshmap/sshmap localhost --runscript testscript.test' + ).read().strip() self.assertEqual('localhost: hello', result) os.remove('testscript.test') def test_shell_script_sudo(self): # Run a ssh command to localhost and verify it works open('testscript.test', 'w').write('#!/bin/bash\nid\n') - result = os.popen('sshmap/sshmap localhost --runscript testscript.test --sudo').read().strip() - self.assert_('localhost: uid=0(root) gid=0(root) groups=0(root)' in result) + result = os.popen( + 'sshmap/sshmap localhost --runscript testscript.test --sudo' + ).read().strip() + self.assert_( + 'localhost: uid=0(root) gid=0(root) groups=0(root)' in result) os.remove('testscript.test') if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 0a5880e9beb9bb7cf934d08f0fc154a326c15b70 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Mon, 22 Jul 2013 14:48:22 -0700 Subject: [PATCH 31/52] Start adding rpc and unit tests --- LICENSE.txt | 12 ------------ sshmap/__init__.py | 4 ++-- sshmap/sshmap.py | 20 ++++++++++++++++++++ tests/test.py | 14 +++++++++++++- 4 files changed, 35 insertions(+), 15 deletions(-) delete mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 4c3ab8f..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1,12 +0,0 @@ - Copyright (c) 2010 Yahoo! Inc. All rights reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. diff --git a/sshmap/__init__.py b/sshmap/__init__.py index 481c85c..e7ef64f 100755 --- a/sshmap/__init__.py +++ b/sshmap/__init__.py @@ -1,8 +1,8 @@ -#Copyright (c) 2012 Yahoo! Inc. All rights reserved. +#Copyright (c) 2012-2013 Yahoo! Inc. All rights reserved. #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. #You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 44b38dc..0ced237 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -29,6 +29,7 @@ import types import random import signal +import inspect import multiprocessing import logging @@ -408,6 +409,25 @@ def init_worker(): signal.signal(signal.SIGINT, signal.SIG_IGN) +def rpc(method, host_range, *args, **kwargs): + """ + Perform a remote procedure call on a range of hosts via ssh. + + This requires the python function/method be in an actual source file. It + will not work from the python shell. + :param method: Python function to execute + :param host_range: List of hosts to execute on + :param args: Arguments + :param kwargs: Keyword arguments + """ + print('Required Arguments:') + print(method) + print(host_range) + print('Method Arguments') + print(*args) + print(**kwargs) + + def run(host_range, command, username=None, password=None, sudo=False, script=None, timeout=None, sort=False, jobs=None, output_callback=callback.summarize_failures, diff --git a/tests/test.py b/tests/test.py index b00d850..0dcb528 100755 --- a/tests/test.py +++ b/tests/test.py @@ -20,6 +20,11 @@ import unittest +def test_rpc_method(): + import os + return os.popen('uname -n') + + class TestSSH(unittest.TestCase): """ sshmap unit tests @@ -48,7 +53,7 @@ def test_shell_script_as_user(self): os.remove('testscript.test') def test_shell_script_sudo(self): - # Run a ssh command to localhost and verify it works + """Run a ssh command to localhost and verify it works """ open('testscript.test', 'w').write('#!/bin/bash\nid\n') result = os.popen( 'sshmap/sshmap localhost --runscript testscript.test --sudo' @@ -57,6 +62,13 @@ def test_shell_script_sudo(self): 'localhost: uid=0(root) gid=0(root) groups=0(root)' in result) os.remove('testscript.test') + def test_rpc_call(self): + """ + Execute an rpc call without arguments via ssh + """ + result = sshmap.rpc(test_rpc_method,['localhost']) + self.assertEqual(result, os.popen('uname -n').read()) + if __name__ == '__main__': unittest.main() From 0dcaf0bd20000546d7145f218a005b586c0816a8 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Mon, 22 Jul 2013 14:55:17 -0700 Subject: [PATCH 32/52] Added missing .gitignore --- .gitignore | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8500dc7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Pycharm +.idea + +# Backup files +*~ From 553f06703e7b8c74efd07206174c8be707e882e3 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Mon, 22 Jul 2013 15:01:56 -0700 Subject: [PATCH 33/52] Add missign import to __init__.py --- sshmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sshmap/__init__.py b/sshmap/__init__.py index e7ef64f..ddfdea8 100755 --- a/sshmap/__init__.py +++ b/sshmap/__init__.py @@ -15,4 +15,4 @@ import sshmap import callback import utility -from sshmap import run, run_command \ No newline at end of file +from sshmap import run, run_command, rpc \ No newline at end of file From 31826fe6717ead002cbf47c4c34d807761af6fb5 Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Mon, 22 Jul 2013 16:59:40 -0700 Subject: [PATCH 34/52] Another attempt at fixing sudo --- sshmap/sshmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 0ced237..1de95c3 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -3,7 +3,7 @@ #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. #You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software From bc1d1cab490750dfb31b4ec54067e6abb6633c7b Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Tue, 23 Jul 2013 08:35:57 -0700 Subject: [PATCH 35/52] runner additions/changes --- .idea/codeStyleSettings.xml | 13 ++++++++++ .idea/vagrant.xml | 8 ++++++ setup.py | 2 +- sshmap/__init__.py | 1 + sshmap/runner.py | 49 +++++++++++++++++++++++++++++++++++++ sshmap/sshmap.py | 25 +++++++++++++------ tests/test.py | 2 +- 7 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 .idea/codeStyleSettings.xml create mode 100644 .idea/vagrant.xml create mode 100644 sshmap/runner.py diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml new file mode 100644 index 0000000..9178b38 --- /dev/null +++ b/.idea/codeStyleSettings.xml @@ -0,0 +1,13 @@ + + + + + + + diff --git a/.idea/vagrant.xml b/.idea/vagrant.xml new file mode 100644 index 0000000..dc044d4 --- /dev/null +++ b/.idea/vagrant.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/setup.py b/setup.py index ca339e5..914f198 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.40", + version="0.5.45", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/__init__.py b/sshmap/__init__.py index ddfdea8..416f80f 100755 --- a/sshmap/__init__.py +++ b/sshmap/__init__.py @@ -15,4 +15,5 @@ import sshmap import callback import utility +import runner from sshmap import run, run_command, rpc \ No newline at end of file diff --git a/sshmap/runner.py b/sshmap/runner.py new file mode 100644 index 0000000..7863dbd --- /dev/null +++ b/sshmap/runner.py @@ -0,0 +1,49 @@ +#Copyright (c) 2012 Yahoo! Inc. All rights reserved. +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +""" +Python rpc remote runner handlers +""" +__author__ = 'dhubbard' +import os +import base64 + + +script_stdin = """import os +os.popen(\"{command}\".decode('base64').decode('{compressor}'),'w').write(\"\"\"{input}\"\"\".decode('base64').decode('{compressor}')) +""" + +script_sudo = """import os +command = \"{command}\".decode('base64').decode('{compressor}') +fh = os.popen(\"sudo -S \" + command,'w') +fh.write(\"{password}\\n\") +fh.write(\"\"\"{input}\"\"\".decode('base64').decode('{compressor}')) +""" + +def get_runner(command, input, password='', sudo=False, compressor='bz2'): + if sudo: + script = script_sudo + else: + script = script_stdin + + if compressor not in ['bz2', 'zlilb']: + compressor = 'bz2' + + base64_command = base64.b64encode(command.encode('bz2')) + base64_input = base64.b64encode(input.encode('bz2')) + + return script.format( + command=base64_command, + input=base64_input, + compressor=compressor, + password=password + ) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 1de95c3..916b3e0 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -41,7 +41,7 @@ import utility import callback import defaults - +import runner # Fix to make ctrl-c correctly terminate child processes # spawned by the multiprocessing module @@ -362,14 +362,20 @@ def run_command(host, command="uname -a", username=None, password=None, import django.conf django.conf.settings.configure() - template = open(script, 'r').read() + if os.path.exists(script): + template = open(script, 'r').read() + else: + template = script if script_parameters: c = django.template.Context({'argv': script_parameters}) else: c = django.template.Context({}) stdin.write(django.template.Template(template).render(c)) except Exception as e: - stdin.write(open(script, 'r').read()) + if os.path.exists(script): + stdin.write(open(script, 'r').read()) + else: + stdin.write(script) stdin.flush() stdin.channel.shutdown_write() try: @@ -384,7 +390,7 @@ def run_command(host, command="uname -a", username=None, password=None, skip = False for el in result.err: if check_prompt: - if password in el or 'assword:' in el: + if password in el or 'assword:' in el or '[sudo] password' in el: skip = True else: check_prompt = False @@ -424,13 +430,18 @@ def rpc(method, host_range, *args, **kwargs): print(method) print(host_range) print('Method Arguments') - print(*args) - print(**kwargs) + print(args) + print(kwargs) + run_script = runner.get_runner(command='/usr/bin/python', input=inspect.getsource(method)) + print('Remote run script') + print(run_script) + os.popen('/usr/bin/python', 'w').write(run_script) + run(host_range=host_range, command='/usr/bin/python', script=run_script) def run(host_range, command, username=None, password=None, sudo=False, script=None, timeout=None, sort=False, - jobs=None, output_callback=callback.summarize_failures, + jobs=None, output_callback=[callback.summarize_failures], parms=None, shuffle=False, chunksize=None): """ Run a command on a hostlists host_range of hosts diff --git a/tests/test.py b/tests/test.py index 0dcb528..50fa518 100755 --- a/tests/test.py +++ b/tests/test.py @@ -56,7 +56,7 @@ def test_shell_script_sudo(self): """Run a ssh command to localhost and verify it works """ open('testscript.test', 'w').write('#!/bin/bash\nid\n') result = os.popen( - 'sshmap/sshmap localhost --runscript testscript.test --sudo' + 'sshmap/sshmap localhost --runscript testscript.test --sudo --timeout 15' ).read().strip() self.assert_( 'localhost: uid=0(root) gid=0(root) groups=0(root)' in result) From 915b71fb7f210f2af5adabeffab9c44ac676a251 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 23 Jul 2013 12:50:30 -0700 Subject: [PATCH 36/52] pep8 cleanup --- sshmap/sshmap | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sshmap/sshmap b/sshmap/sshmap index 73b39df..4eacb91 100755 --- a/sshmap/sshmap +++ b/sshmap/sshmap @@ -25,7 +25,8 @@ import hostlists if __name__ == "__main__": parser = optparse.OptionParser() #parser.add_option("--html", dest="html", default=False,action="store_true", help="Use HTML for formatting") - parser.add_option("--output_json", dest = "output_json", default = False, action = "store_true", + parser.add_option("--output_json", dest="output_json", + default=False, action="store_true", help = "Output in JSON format") parser.add_option("--output_base64", dest = "output_base64", default = False, action = "store_true", help = "Output in base64 format") From 56793a02d6fadcba2c0a87878b4241d00f3e2543 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 23 Jul 2013 12:52:53 -0700 Subject: [PATCH 37/52] pep8 cleanup --- tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 50fa518..9292b6c 100755 --- a/tests/test.py +++ b/tests/test.py @@ -66,7 +66,7 @@ def test_rpc_call(self): """ Execute an rpc call without arguments via ssh """ - result = sshmap.rpc(test_rpc_method,['localhost']) + result = sshmap.rpc(test_rpc_method, ['localhost']) self.assertEqual(result, os.popen('uname -n').read()) From 9933c5993d6c4a92b1ba89a1ae1854012eb8feb6 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 23 Jul 2013 15:06:02 -0700 Subject: [PATCH 38/52] Use a runner --- sshmap/__init__.py | 2 +- sshmap/runner.py | 11 +++++------ sshmap/sshmap.py | 36 +++++++++++++++--------------------- tests/test.py | 17 +++++++++-------- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/sshmap/__init__.py b/sshmap/__init__.py index 416f80f..f492023 100755 --- a/sshmap/__init__.py +++ b/sshmap/__init__.py @@ -16,4 +16,4 @@ import callback import utility import runner -from sshmap import run, run_command, rpc \ No newline at end of file +from sshmap import run, run_command \ No newline at end of file diff --git a/sshmap/runner.py b/sshmap/runner.py index 7863dbd..7e5d1b3 100644 --- a/sshmap/runner.py +++ b/sshmap/runner.py @@ -29,11 +29,10 @@ fh.write(\"\"\"{input}\"\"\".decode('base64').decode('{compressor}')) """ -def get_runner(command, input, password='', sudo=False, compressor='bz2'): - if sudo: - script = script_sudo - else: - script = script_stdin +def get_runner(command, input, password='', runner_script=None, + compressor='bz2'): + if not runner_script: + runner_script = script_stdin if compressor not in ['bz2', 'zlilb']: compressor = 'bz2' @@ -41,7 +40,7 @@ def get_runner(command, input, password='', sudo=False, compressor='bz2'): base64_command = base64.b64encode(command.encode('bz2')) base64_input = base64.b64encode(input.encode('bz2')) - return script.format( + return runner_script.format( command=base64_command, input=base64_input, compressor=compressor, diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 916b3e0..58651ab 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -390,7 +390,8 @@ def run_command(host, command="uname -a", username=None, password=None, skip = False for el in result.err: if check_prompt: - if password in el or 'assword:' in el or '[sudo] password' in el: + if password in el or 'assword:' in el or \ + '[sudo] password' in el: skip = True else: check_prompt = False @@ -415,28 +416,21 @@ def init_worker(): signal.signal(signal.SIGINT, signal.SIG_IGN) -def rpc(method, host_range, *args, **kwargs): +def run_with_runner(*args, **kwargs): """ - Perform a remote procedure call on a range of hosts via ssh. - - This requires the python function/method be in an actual source file. It - will not work from the python shell. - :param method: Python function to execute - :param host_range: List of hosts to execute on - :param args: Arguments - :param kwargs: Keyword arguments + Run a command with a python runner script """ - print('Required Arguments:') - print(method) - print(host_range) - print('Method Arguments') - print(args) - print(kwargs) - run_script = runner.get_runner(command='/usr/bin/python', input=inspect.getsource(method)) - print('Remote run script') - print(run_script) - os.popen('/usr/bin/python', 'w').write(run_script) - run(host_range=host_range, command='/usr/bin/python', script=run_script) + if 'runner' in kwargs.keys() and isinstance( + kwargs['runner'], type.FunctionType): + kwargs['script'] = runner.get_runner( + command=args[1], + input="", + password=kwargs['password'], + runner_script=kwargs['runner'], + compressor='bz2' + ) + del kwargs['runner'] + return run(*args, **kwargs) def run(host_range, command, username=None, password=None, sudo=False, diff --git a/tests/test.py b/tests/test.py index 9292b6c..ac7c95c 100755 --- a/tests/test.py +++ b/tests/test.py @@ -18,11 +18,7 @@ import sshmap import os import unittest - - -def test_rpc_method(): - import os - return os.popen('uname -n') +import runner class TestSSH(unittest.TestCase): @@ -56,17 +52,22 @@ def test_shell_script_sudo(self): """Run a ssh command to localhost and verify it works """ open('testscript.test', 'w').write('#!/bin/bash\nid\n') result = os.popen( - 'sshmap/sshmap localhost --runscript testscript.test --sudo --timeout 15' + 'sshmap/sshmap localhost --runscript testscript.test --sudo ' + '--timeout 15' ).read().strip() self.assert_( 'localhost: uid=0(root) gid=0(root) groups=0(root)' in result) os.remove('testscript.test') - def test_rpc_call(self): + def run_with_runner(self): """ Execute an rpc call without arguments via ssh """ - result = sshmap.rpc(test_rpc_method, ['localhost']) + result = sshmap.run_with_runner( + 'localhost', + 'uname -n', + runner=runner.script_stdin + ) self.assertEqual(result, os.popen('uname -n').read()) From 5f3ed33a0eac25a1808412e85e679b5e5e3c7f6e Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 23 Jul 2013 15:48:01 -0700 Subject: [PATCH 39/52] Fix the test --- tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index ac7c95c..6a5a73d 100755 --- a/tests/test.py +++ b/tests/test.py @@ -66,7 +66,7 @@ def run_with_runner(self): result = sshmap.run_with_runner( 'localhost', 'uname -n', - runner=runner.script_stdin + runner=sshmap.runner.script_stdin ) self.assertEqual(result, os.popen('uname -n').read()) From e435504812f36e4dfadea68c1c5652d6a70c404c Mon Sep 17 00:00:00 2001 From: dhubbard Date: Tue, 23 Jul 2013 15:49:09 -0700 Subject: [PATCH 40/52] Fix imports --- tests/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 6a5a73d..d126442 100755 --- a/tests/test.py +++ b/tests/test.py @@ -18,7 +18,6 @@ import sshmap import os import unittest -import runner class TestSSH(unittest.TestCase): From e58a1d70e01318ec0e7756ca5d5037cb2db04379 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 09:51:12 -0700 Subject: [PATCH 41/52] Have the remove passwords code ignore blank lines --- sshmap/sshmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 58651ab..5ef2661 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -391,7 +391,7 @@ def run_command(host, command="uname -a", username=None, password=None, for el in result.err: if check_prompt: if password in el or 'assword:' in el or \ - '[sudo] password' in el: + '[sudo] password' in el or el.strip() == '': skip = True else: check_prompt = False From ba0126536098f8762d65a413f71bb017e2218a8a Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 09:55:43 -0700 Subject: [PATCH 42/52] Remove the sudo prompt message from the stdout stream when using sudo --- sshmap/defaults.py | 8 ++++++++ sshmap/sshmap.py | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sshmap/defaults.py b/sshmap/defaults.py index 191b09a..3b03399 100644 --- a/sshmap/defaults.py +++ b/sshmap/defaults.py @@ -57,3 +57,11 @@ "port": "6667", "use_ssl": False, } + +sudo_message = [ + 'We trust you have received the usual lecture from the local System', + 'Administrator. It usually boils down to these three things:', + '#1) Respect the privacy of others.', + '#2) Think before you type.', + '#3) With great power comes great responsibility.' +] diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 5ef2661..8d0d27e 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -391,7 +391,8 @@ def run_command(host, command="uname -a", username=None, password=None, for el in result.err: if check_prompt: if password in el or 'assword:' in el or \ - '[sudo] password' in el or el.strip() == '': + '[sudo] password' in el or el.strip() == '' or \ + el.strip() in defaults.sudo_message: skip = True else: check_prompt = False From 8be72be714830257dc273d39d2d12d8833ecc3d6 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 09:56:17 -0700 Subject: [PATCH 43/52] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 914f198..0d445ce 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.45", + version="0.5.46", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", From 60996847ca0018c996623da4e4aa7c9f8a55dbc4 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 10:28:03 -0700 Subject: [PATCH 44/52] Remove disabled code and PEP8 cleanups --- sshmap/sshmap | 46 +++++++++++++++++++++++----------------------- sshmap/sshmap.py | 28 ++++------------------------ 2 files changed, 27 insertions(+), 47 deletions(-) diff --git a/sshmap/sshmap b/sshmap/sshmap index 4eacb91..4b13aed 100755 --- a/sshmap/sshmap +++ b/sshmap/sshmap @@ -3,7 +3,7 @@ #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. #You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software @@ -24,12 +24,11 @@ import hostlists if __name__ == "__main__": parser = optparse.OptionParser() - #parser.add_option("--html", dest="html", default=False,action="store_true", help="Use HTML for formatting") parser.add_option("--output_json", dest="output_json", default=False, action="store_true", help = "Output in JSON format") - parser.add_option("--output_base64", dest = "output_base64", default = False, action = "store_true", - help = "Output in base64 format") + parser.add_option("--output_base64", dest="output_base64", default=False, + action="store_true", help="Output in base64 format") parser.add_option("--summarize_failed", dest = "summarize_failed", default = False, action = "store_true", help = "Print a list of hosts that failed at the end") parser.add_option("--aggregate_output", "--collapse", dest = "aggregate_output", default = False, @@ -55,8 +54,8 @@ if __name__ == "__main__": help = "Don't show a status count as the command progresses") parser.add_option("--sudo", dest = "sudo", default = False, action = "store_true", help = "Use sudo to run the command as root") - parser.add_option("--password", dest = "password", default = None, action = "store_true", - help = "Prompt for a password") + parser.add_option("--password", dest="password", default=None, + action="store_true", help="Prompt for a password") (options, args) = parser.parse_args() @@ -66,14 +65,6 @@ if __name__ == "__main__": command = firstline[2:] args.append(command) - #if len(args) != 2: - # if len(args) and args[0] in ['test']: - # Unit testing - # import doctest - - # doctest.testmod() - #sys.exit(0) - # Default option values options.password = None options.username = getpass.getuser() @@ -99,21 +90,27 @@ if __name__ == "__main__": if options.sudo: # Prompt for password, we really need to add a password file option try: - options.password = os.environ['OS_PASSWORD'] + options.password = os.environ['SUDO_PASSWORD'] except KeyError: - options.password = getpass.getpass('Enter sudo password for user ' + getpass.getuser() + ': ') + options.password = getpass.getpass( + 'Enter sudo password for user ' + getpass.getuser() + ': ') elif options.password: # Prompt for password, we really need to add a password file option try: - options.password = os.environ['OS_PASSWORD'] + options.password = os.environ['SUDO_PASSWORD'] except KeyError: - options.password = getpass.getpass('Enter password for user ' + getpass.getuser() + ': ') + options.password = getpass.getpass( + 'Enter password for user ' + getpass.getuser() + ': ') command = ' '.join(args[1:]) range = args[0] - results = sshmap.run(args[0], command, username=options.username, password=options.password, sudo=options.sudo, - timeout=options.timeout, script=options.runscript, jobs=options.jobs, sort=options.sort, - shuffle=options.shuffle, output_callback=callback, parms=vars(options)) + results = sshmap.run( + args[0], command, username=options.username, + password=options.password, sudo=options.sudo, + timeout=options.timeout, script=options.runscript, jobs=options.jobs, + sort=options.sort, + shuffle=options.shuffle, output_callback=callback, parms=vars(options) + ) if options.aggregate_output: aggregate_hosts = results.setting('aggregate_hosts') collapsed_output = results.setting('collapsed_output') @@ -126,5 +123,8 @@ if __name__ == "__main__": stdout, stderr = collapsed_output[md5] if len(stdout): print ''.join(stdout) if len(stderr): print >> sys.stderr, '\n'.join(stderr) - if options.summarize_failed and 'failures' in results.parm.keys() and len(results.parm['failures']): - print 'SSH Failed to: %s' % hostlists.compress(results.parm['failures']) + if options.summarize_failed and 'failures' in results.parm.keys() and \ + len(results.parm['failures']): + print( + 'SSH Failed to: %s' % hostlists.compress(results.parm['failures']) + ) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 8d0d27e..0a5dbd1 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -327,33 +327,12 @@ def run_command(host, command="uname -a", username=None, password=None, # Send the password stdin.write(password + '\r') stdin.flush() - - if False: - # Remove the password prompt and password from the output - # should only be needed if using a pty - prompt = _term_readline(stdout) - seen_password = False - seen_password_prompt = False - #print 'READ:',prompt - while 'assword:' in prompt or False or password in prompt or \ - 'try again' in prompt or len(prompt.strip()) == 0: - if 'try again' in prompt: - result.ssh_retcode = defaults.RUN_FAIL_BADPASSWORD - return result - prompt_new = _term_readline(stdout) - if 'assword:' in prompt: - seen_password_prompt = True - if password in prompt: - seen_password = True - if seen_password_prompt or seen_password: - break - prompt = prompt_new except socket.timeout: result.err = ['Timeout during sudo connect, likely bad password'] result.ssh_retcode = defaults.RUN_FAIL_TIMEOUT return result if script: - # Pass the script over stdin and close the channel so the receving end + # Pass the script over stdin and close the channel so the receiving end # gets an EOF process it as a django template with the arguments passed # noinspection PyBroadException try: @@ -371,7 +350,7 @@ def run_command(host, command="uname -a", username=None, password=None, else: c = django.template.Context({}) stdin.write(django.template.Template(template).render(c)) - except Exception as e: + except: if os.path.exists(script): stdin.write(open(script, 'r').read()) else: @@ -401,7 +380,6 @@ def run_command(host, command="uname -a", username=None, password=None, skip = False result.err = err - #print result.err result.retcode = chan.recv_exit_status() if close_client: client.close() @@ -420,6 +398,8 @@ def init_worker(): def run_with_runner(*args, **kwargs): """ Run a command with a python runner script + :param args: + :param kwargs: """ if 'runner' in kwargs.keys() and isinstance( kwargs['runner'], type.FunctionType): From d344538f62dcff77c2670958011ac30ceada8b85 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 10:29:02 -0700 Subject: [PATCH 45/52] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0d445ce..93fb6a3 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.46", + version="0.5.47", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", From cca24d9b2f20f5baaa7cb670c27785e91554224c Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 10:34:23 -0700 Subject: [PATCH 46/52] Pep8 cleanups --- sshmap/sshmap | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sshmap/sshmap b/sshmap/sshmap index 4b13aed..1bccaf4 100755 --- a/sshmap/sshmap +++ b/sshmap/sshmap @@ -52,9 +52,10 @@ if __name__ == "__main__": help = "Script to process the output of each host. The hostname will be passed as the first argument and the stdin/stderr from the host will be passed as stdin/stderr of the script") parser.add_option("--no_status", dest = "show_status", default = True, action = "store_false", help = "Don't show a status count as the command progresses") - parser.add_option("--sudo", dest = "sudo", default = False, action = "store_true", + parser.add_option("--sudo", dest="sudo", default=False, + action="store_true", help = "Use sudo to run the command as root") - parser.add_option("--password", dest="password", default=None, + parser.add_option("--password", dest="password", default=False, action="store_true", help="Prompt for a password") (options, args) = parser.parse_args() @@ -121,8 +122,10 @@ if __name__ == "__main__": print ','.join(hostlists.compress(aggregate_hosts[md5])) print "-" * (int(columns)-2) stdout, stderr = collapsed_output[md5] - if len(stdout): print ''.join(stdout) - if len(stderr): print >> sys.stderr, '\n'.join(stderr) + if len(stdout): + print ''.join(stdout) + if len(stderr): + print >> sys.stderr, '\n'.join(stderr) if options.summarize_failed and 'failures' in results.parm.keys() and \ len(results.parm['failures']): print( From 7fe4ef63f7a0e457202beb90014546ec1296473f Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 10:41:34 -0700 Subject: [PATCH 47/52] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 93fb6a3..5390100 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.47", + version="0.5.48", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", From e9294749addf23a3eb718d0fa97323fd697e29fa Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 10:50:04 -0700 Subject: [PATCH 48/52] Change the env variable for the sudo password --- sshmap/sshmap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sshmap/sshmap b/sshmap/sshmap index 1bccaf4..745eabf 100755 --- a/sshmap/sshmap +++ b/sshmap/sshmap @@ -91,14 +91,14 @@ if __name__ == "__main__": if options.sudo: # Prompt for password, we really need to add a password file option try: - options.password = os.environ['SUDO_PASSWORD'] + options.password = os.environ['SSHMAP_SUDO_PASSWORD'] except KeyError: options.password = getpass.getpass( 'Enter sudo password for user ' + getpass.getuser() + ': ') elif options.password: # Prompt for password, we really need to add a password file option try: - options.password = os.environ['SUDO_PASSWORD'] + options.password = os.environ['SSHMAP_SUDO_PASSWORD'] except KeyError: options.password = getpass.getpass( 'Enter password for user ' + getpass.getuser() + ': ') From 6132c5ee7e26247348b655b66daf305c7c9af370 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 17:46:06 -0700 Subject: [PATCH 49/52] Add some from imports to pull in callbacks into their old name in the sshmap namespace. --- sshmap/sshmap.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index 0a5dbd1..ebba97c 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -43,6 +43,13 @@ import defaults import runner +# For backwards compatibility +from callback import summarize_failures as callback_summarize_failures +from callback import aggregate_output as callback_aggregate_output +from callback import exec_command as callback_exec_command +from callback import filter_match as callback_filter_match +from callback import status_count as callback_status_count + # Fix to make ctrl-c correctly terminate child processes # spawned by the multiprocessing module from multiprocessing.pool import IMapIterator From c60613e54c35ebdd29844ac41a87d85f2f8545d0 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 17:46:35 -0700 Subject: [PATCH 50/52] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5390100..6c6a9e9 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.48", + version="0.5.49", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", From d97f06400c8f47d8113f12975643728f3b464b02 Mon Sep 17 00:00:00 2001 From: dhubbard Date: Wed, 24 Jul 2013 17:50:50 -0700 Subject: [PATCH 51/52] Move backwards compat stuff to __init__.py --- setup.py | 2 +- sshmap/__init__.py | 11 ++++++++++- sshmap/sshmap.py | 7 ------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 6c6a9e9..f9591a9 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="sshmap", - version="0.5.49", + version="0.5.50", author="Dwight Hubbard", author_email="dhubbard@yahoo-inc.com", url="https://github.com/dwighthubbard/sshmap", diff --git a/sshmap/__init__.py b/sshmap/__init__.py index f492023..fa25fe7 100755 --- a/sshmap/__init__.py +++ b/sshmap/__init__.py @@ -16,4 +16,13 @@ import callback import utility import runner -from sshmap import run, run_command \ No newline at end of file + +# For backwards compatibility +from callback import summarize_failures as callback_summarize_failures +from callback import aggregate_output as callback_aggregate_output +from callback import exec_command as callback_exec_command +from callback import filter_match as callback_filter_match +from callback import status_count as callback_status_count + +# The actual used sshmap functions +from sshmap import run, run_command diff --git a/sshmap/sshmap.py b/sshmap/sshmap.py index ebba97c..0a5dbd1 100755 --- a/sshmap/sshmap.py +++ b/sshmap/sshmap.py @@ -43,13 +43,6 @@ import defaults import runner -# For backwards compatibility -from callback import summarize_failures as callback_summarize_failures -from callback import aggregate_output as callback_aggregate_output -from callback import exec_command as callback_exec_command -from callback import filter_match as callback_filter_match -from callback import status_count as callback_status_count - # Fix to make ctrl-c correctly terminate child processes # spawned by the multiprocessing module from multiprocessing.pool import IMapIterator From fb5487bab827226460b12118ec6bae3fa63fde06 Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Fri, 26 Jul 2013 22:47:36 +0000 Subject: [PATCH 52/52] Remove junk .idea directory --- .idea/codeStyleSettings.xml | 13 ------------- .idea/vagrant.xml | 8 -------- 2 files changed, 21 deletions(-) delete mode 100644 .idea/codeStyleSettings.xml delete mode 100644 .idea/vagrant.xml diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml deleted file mode 100644 index 9178b38..0000000 --- a/.idea/codeStyleSettings.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - diff --git a/.idea/vagrant.xml b/.idea/vagrant.xml deleted file mode 100644 index dc044d4..0000000 --- a/.idea/vagrant.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - -