From 4249295be354f343a19bf903070b4996baeb53ce Mon Sep 17 00:00:00 2001 From: Paul Nelson Date: Thu, 6 Sep 2018 13:01:43 -0400 Subject: [PATCH] support sqlalchemy connection checks since mere port availability wasn't always sufficient --- .cmd.yml | 2 ++ README.md | 29 ++++++++++++++++++------- no_you_talk_to_the_hand.py | 44 ++++++++++++++++++++++++++------------ requirements.txt | 2 +- setup.py | 22 ++++++++----------- 5 files changed, 63 insertions(+), 36 deletions(-) diff --git a/.cmd.yml b/.cmd.yml index 516e9eb..accec72 100644 --- a/.cmd.yml +++ b/.cmd.yml @@ -18,7 +18,9 @@ install: package: - rm -rf build - rm -rf dist + - pandoc -o README.rst README.md - python setup.py sdist bdist_wheel + - rm README.rst publish: - twine upload -r pypi -u flashashen dist/{{app_name}}* diff --git a/README.md b/README.md index 348e398..b70292c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # no, *YOU* talk to the hand! ![CircleCI](https://circleci.com/gh/flashashen/no-YOU-talk-to-the-hand.svg?style=svg) -[![PyPI version](https://badge.fury.io/py/no-you-talk-to-the-hand.svg)](https://badge.fury.io/py/no-you-talk-to-the-hand) ![Python versions](https://img.shields.io/pypi/pyversions/no-YOU-talk-to-the-hand.svg) ![MIT License](https://img.shields.io/github/license/flashashen/no-YOU-talk-to-the-hand.svg) @@ -11,6 +10,7 @@ Access all your corporate stuff and web stuff at the same time without fuss **You want this if you're being worn out by:** + - Seeing the 'talk to the hand' page from the corporate web proxy/filter - Re-entering ssh credentials over and over (key based auth isn't allowed everywhere) - Tunnel/proxy setup in too many places and in too many ways @@ -18,7 +18,7 @@ Access all your corporate stuff and web stuff at the same time without fuss - Tunnels dropping silently - Forgetting to manually bring up tunnels after logging onto vpn -no-YOU-talk-to-the-hand really has **solved all these issues** for me, by providing a straight-forward combination of [sshuttle](https://github.com/sshuttle/sshuttle) for the heavy network lifting, supervisord to keep everything up and manageable, and yaml to keep it simple and organized +no-YOU-talk-to-the-hand solves all these issues** by providing a straight-forward combination of [sshuttle](https://github.com/sshuttle/sshuttle) for the heavy network lifting, supervisord to keep everything up and manageable, and yaml to keep it simple and organized Works with Linux and MacOS but **not MS Windows** due to sshuttle though there is a workaround for windows described [here](http://sshuttle.readthedocs.io/en/stable/windows.html) @@ -35,7 +35,7 @@ Works with Linux and MacOS but **not MS Windows** due to sshuttle though there i - Supportes *nested dependencies*. For example: (qa_db, prod_db) -- depends --> (corp_private) -- depends --> (corp_vpn) -### Yaml replaces all your tunnel scripts/aliases, ssh setup inside db tools, application specific web proxy setup, etc. +### Config.yml replaces all your tunnel scripts/aliases, ssh setup inside db tools, application specific web proxy setup, etc: ```yaml @@ -126,6 +126,13 @@ tunnels: host: *HOST_CORP_PRIVILEGED_APP user: *CORP_USER pass: *CORP_PASS + check: + driver: mysql+pymysql + db: testdb + user: testuser + pass: testpass + host: 10.0.2.1 + port: '3306' forwards: # includes and excludes. items can be ips, subnets, or lists of ip/subnets. include: @@ -140,7 +147,13 @@ tunnels: ```pip install no_you_talk_to_the_hand``` -*Note* If you configure a password for any remote server then [sshpass](https://gist.github.com/arunoda/7790979) is required +If you're getting install errors like 'TLSV1_ALERT_PROTOCOL_VERSION' you may first need to upgrade pip + +```curl https://bootstrap.pypa.io/get-pip.py | python``` + +If you configure a password for any remote server then [sshpass](https://gist.github.com/arunoda/7790979) is required + +If you check a tunnel via a sqlalchemy connection (see prod_db tunnel in sample config above) then sqlalchemy and the appropriate driver must be installed separately ## Running @@ -149,21 +162,21 @@ Below are some sample commands. **Note:** Before running a configuration file called config.yml must be created in the project directory. Look at sample_config.yml as a start. -#### Start - Start the supervisord process and begin managing the configured tunnels +### Start - Start the supervisord process and begin managing the configured tunnels ``` $ nyttth start ``` -#### Stop - Stop supervisord process and all tunnels with it +### Stop - Stop supervisord process and all tunnels with it ``` $ nyttth stop ``` -#### Status - View status of all defined tunnels +### Status - View status of all defined tunnels when VPN is down: @@ -195,7 +208,7 @@ itun vpn RUNNING up qadb vpn RUNNING up ``` -#### Tail - Tail the tunnel monitor that checks tunnel statuses and brings them up or down as needed. +### Tail - Tail the tunnel monitor that checks tunnel statuses and brings them up or down as needed. when VPN Disconnects: diff --git a/no_you_talk_to_the_hand.py b/no_you_talk_to_the_hand.py index 638d94d..01d7f5e 100755 --- a/no_you_talk_to_the_hand.py +++ b/no_you_talk_to_the_hand.py @@ -8,6 +8,8 @@ import click import jinja2 +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) import logging logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s: %(message)s') @@ -191,6 +193,8 @@ def get_check_type(tun_chk_cfg): if not tun_chk_cfg: return 'supervisor' + elif 'driver' in tun_chk_cfg: + return 'db' elif 'url' in tun_chk_cfg and tun_chk_cfg['url']: protocol = tun_chk_cfg['url'].split('://')[0] if 'http' in protocol: @@ -210,11 +214,15 @@ def dbengine_create_func(config): import sqlalchemy params = config.copy() - if not 'poolclass' in params: + if 'poolclass' not in params: params['poolclass'] = sqlalchemy.pool.NullPool - if not 'connect_timeout' in params: - params['connect_timeout'] = 10 - url = params.pop('url') + url = "{:s}://{:s}:{:s}@{:s}:{:s}/{:s}?".format( + params.pop('driver'), + params.pop('user'), + params.pop('pass'), + params.pop('host'), + params.pop('port'), + params.pop('db')) return sqlalchemy.create_engine(url, **params) @@ -232,18 +240,23 @@ def check_tunnel(tunnel_name): result = 'up' if proc_started(get_supervisor().getProcessInfo(tunnel_name)) else 'down' elif ctype == 'url': rsp = requests.head(chk['url'], verify=False, timeout=timeout) - log.debug('{} check ok: {} {}'.format(tunnel_name, chk, rsp)) + ok = rsp.status_code >= 200 and rsp.status_code < 300 + log.debug('{} check {}: {} {}'.format( + tunnel_name, + 'ok' if ok else "failed", + chk, + rsp)) result = 'up' if rsp.status_code >= 200 and rsp.status_code < 300 else 'down' elif ctype == 'db': import sqlalchemy - eng = get_run_data(tunnel_name)['db_engine']; + eng = get_run_data(tunnel_name).get('db_engine'); if not eng: eng = dbengine_create_func(chk) get_run_data(tunnel_name)['db_engine'] = eng conn = eng.connect() conn.close() log.debug('{} check ok: {}'.format(tunnel_name, chk)) - result = 'up' + result = 'up' elif ctype == 'socket': s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout) @@ -254,6 +267,7 @@ def check_tunnel(tunnel_name): else: result = 'invalid' except Exception as e: + log.error(str(e)) log.debug('{} check failed: {} {}'.format(tunnel_name, chk, e)) result = 'down' @@ -296,21 +310,22 @@ def stop_dependent_tunnels(tunnel_name): stop_dependent_tunnels(tunnel) -def check_dependent_tunnels(check_result_parent = None, check_only=False): +def check_dependent_tunnels(check_result_parent=None, check_only=False): tunnels = get_tunnel_dependents( check_result_parent.name if check_result_parent else None); if not tunnels: - # log.debug('no dependents\n') + # log.debug('check_dependent_tunnels: no dependents\n') return {} + log.debug('check_dependent_tunnels: ' + ','.join(tunnels)) statuses = [] - if check_result_parent and not check_result_parent.status == 'up': + if check_result_parent and check_result_parent.status != 'up': # If parent is not up, then skip children - statuses = [CheckResult(tunnel_name,'skipped',get_check_type(get_check_cfg(tunnel_name))) - for tunnel_name in tunnels ] + statuses = [CheckResult(tunnel_name, 'skipped', get_check_type(get_check_cfg(tunnel_name))) + for tunnel_name in tunnels] else: # Use a thread pool to perform the checks of child tunnels with futures.ThreadPoolExecutor(max_workers=10) as executor: @@ -329,6 +344,7 @@ def check_dependent_tunnels(check_result_parent = None, check_only=False): if check_result.status != 'down' or check_only: statuses.extend(check_dependent_tunnels(check_result, check_only)) else: + pass # Make sure dependent tunnels are stopped. Don't bother gathering status as we may no longer use it stop_dependent_tunnels(check_result.name) @@ -360,7 +376,7 @@ def vpnmonitor(): results = check_dependent_tunnels() # for result in results: # log.debug('{} check: {}'.format(result.name, result.status)) - time.sleep(6) + time.sleep(20) # childutils.listener.ok() except Exception as e: print e @@ -468,7 +484,7 @@ def get_tunnel_dependents(tunnel_name=None): def get_run_data(tunnel_name): - data = get_config()['tunnels'][tunnel_name]['__run_data']; + data = get_config()['tunnels'][tunnel_name].get('__run_data'); if not data: data = {} get_config()['tunnels'][tunnel_name]['__run_data'] = data diff --git a/requirements.txt b/requirements.txt index b5fc48f..5a5b5c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ supervisor==3.3.1 requests==2.10.0 Jinja2==2.8 futures==3.0.5 -sshuttle===0.78.3 +sshuttle===0.78.1 diff --git a/setup.py b/setup.py index b4c07ab..c72f3a2 100644 --- a/setup.py +++ b/setup.py @@ -1,21 +1,17 @@ # https://packaging.python.org/tutorials/distributing-packages/ from setuptools import setup -# import pypandoc - -# readme = open('README_GENERATED.rst', 'r') -# README_TEXT = pypandoc.convert_file('README.md', 'rst') -# readme.close() - -import os -readme = open('README.md', 'r') -README_TEXT = readme.read() -readme.close() +try: + readme = open('README.rst', 'r') + README_TEXT = readme.read() + readme.close() +except: + README_TEXT = "" setup( name='no_you_talk_to_the_hand', - version='1.0.3', + version='1.0.4', author='flashashen', author_email='flashashen@gmail.com', description='nyttth: Boss your corporate network effortlessly. Automatic and organized tunneling with sshuttle + supervisord + yaml', @@ -33,7 +29,7 @@ ], platforms='osx,linux', - keywords = "ssh vpn tunnel forward daemon", + keywords="ssh vpn tunnel forward daemon", long_description=README_TEXT, py_modules=['no_you_talk_to_the_hand'], install_requires=[ @@ -44,7 +40,7 @@ 'six', 'requests', 'futures', - 'sshuttle==0.78.4' + 'sshuttle==0.78.1' ], entry_points='''