support sqlalchemy connection checks since mere port availability was…

…n't always sufficient

.cmd.yml

@@ -18,7 +18,9 @@ install:
 package:
   - rm -rf build
   - rm -rf dist
+  - pandoc -o README.rst README.md
   - python setup.py sdist bdist_wheel
+  - rm README.rst
 
 publish:
   - twine upload -r pypi -u flashashen dist/{{app_name}}* 

README.md

@@ -1,7 +1,6 @@
 # no, *YOU* talk to the hand!
 
 ![CircleCI](https://circleci.com/gh/flashashen/no-YOU-talk-to-the-hand.svg?style=svg)
-[![PyPI version](https://badge.fury.io/py/no-you-talk-to-the-hand.svg)](https://badge.fury.io/py/no-you-talk-to-the-hand)
 ![Python versions](https://img.shields.io/pypi/pyversions/no-YOU-talk-to-the-hand.svg)
 ![MIT License](https://img.shields.io/github/license/flashashen/no-YOU-talk-to-the-hand.svg)
 
@@ -11,14 +10,15 @@ Access all your corporate stuff and web stuff at the same time without fuss
 
 
 **You want this if you're being worn out by:**
+
 - Seeing the 'talk to the hand' page from the corporate web proxy/filter
 - Re-entering ssh credentials over and over (key based auth isn't allowed everywhere)
 - Tunnel/proxy setup in too many places and in too many ways
 - Not having application specific tunnels (dbvis, datagrip) available from the console
 - Tunnels dropping silently
 - Forgetting to manually bring up tunnels after logging onto vpn
 
-no-YOU-talk-to-the-hand really has **solved all these issues** for me, by providing a straight-forward combination of [sshuttle](https://github.com/sshuttle/sshuttle) for the heavy network lifting, supervisord to keep everything up and manageable, and yaml to keep it simple and organized
+no-YOU-talk-to-the-hand solves all these issues** by providing a straight-forward combination of [sshuttle](https://github.com/sshuttle/sshuttle) for the heavy network lifting, supervisord to keep everything up and manageable, and yaml to keep it simple and organized
 
 Works with Linux and MacOS but **not MS Windows** due to sshuttle though there is a workaround for windows described [here](http://sshuttle.readthedocs.io/en/stable/windows.html)
 
@@ -35,7 +35,7 @@ Works with Linux and MacOS but **not MS Windows** due to sshuttle though there i
 - Supportes *nested dependencies*. For example: (qa_db, prod_db) -- depends --> (corp_private) -- depends --> (corp_vpn)
 
 
-### Yaml replaces all your tunnel scripts/aliases, ssh setup inside db tools, application specific web proxy setup, etc.
+### Config.yml replaces all your tunnel scripts/aliases, ssh setup inside db tools, application specific web proxy setup, etc:
 
 ```yaml
 
@@ -126,6 +126,13 @@ tunnels:
             host: *HOST_CORP_PRIVILEGED_APP
             user: *CORP_USER
             pass: *CORP_PASS
+        check:
+            driver: mysql+pymysql
+            db:   testdb
+            user: testuser
+            pass: testpass
+            host: 10.0.2.1
+            port: '3306'
         forwards:
             # includes and excludes. items can be ips, subnets, or lists of ip/subnets.
             include:
@@ -140,7 +147,13 @@ tunnels:
 ```pip install no_you_talk_to_the_hand```
 
 
-*Note* If you configure a password for any remote server then [sshpass](https://gist.github.com/arunoda/7790979) is required
+If you're getting install errors like 'TLSV1_ALERT_PROTOCOL_VERSION' you may first need to upgrade pip 
+
+```curl https://bootstrap.pypa.io/get-pip.py | python```
+
+If you configure a password for any remote server then [sshpass](https://gist.github.com/arunoda/7790979) is required
+
+If you check a tunnel via a sqlalchemy connection (see prod_db tunnel in sample config above) then sqlalchemy and the appropriate driver must be installed separately
 
 ## Running
 
@@ -149,21 +162,21 @@ Below are some sample commands.
 **Note:** Before running a configuration file called config.yml must be created in the project directory. Look at sample_config.yml as a start.
 
 
-#### Start - Start the supervisord process and begin managing the configured tunnels
+### Start - Start the supervisord process and begin managing the configured tunnels
 
 ```
 $ nyttth start
 ```
 
 
-#### Stop - Stop supervisord process and all tunnels with it
+### Stop - Stop supervisord process and all tunnels with it
 
 ```
 $ nyttth stop
 ```
 
 
-#### Status - View status of all defined tunnels
+### Status - View status of all defined tunnels
 
 when VPN is down:
 
@@ -195,7 +208,7 @@ itun      vpn       RUNNING   up
 qadb      vpn       RUNNING   up        
 ```
 
-#### Tail - Tail the tunnel monitor that checks tunnel statuses and brings them up or down as needed.
+### Tail - Tail the tunnel monitor that checks tunnel statuses and brings them up or down as needed.
 
 when VPN Disconnects:
 

no_you_talk_to_the_hand.py

@@ -8,6 +8,8 @@
 import click
 import jinja2
 
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 import logging
 logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s: %(message)s')
@@ -191,6 +193,8 @@ def get_check_type(tun_chk_cfg):
 
     if not tun_chk_cfg:
         return 'supervisor'
+    elif 'driver' in tun_chk_cfg:
+        return 'db'
     elif 'url' in tun_chk_cfg and tun_chk_cfg['url']:
         protocol = tun_chk_cfg['url'].split('://')[0]
         if 'http' in protocol:
@@ -210,11 +214,15 @@ def dbengine_create_func(config):
 
     import sqlalchemy
     params = config.copy()
-    if not 'poolclass' in params:
+    if 'poolclass' not in params:
         params['poolclass'] = sqlalchemy.pool.NullPool
-    if not 'connect_timeout' in params:
-        params['connect_timeout'] = 10
-    url = params.pop('url')
+    url = "{:s}://{:s}:{:s}@{:s}:{:s}/{:s}?".format(
+        params.pop('driver'),
+        params.pop('user'),
+        params.pop('pass'),
+        params.pop('host'),
+        params.pop('port'),
+        params.pop('db'))
     return sqlalchemy.create_engine(url, **params)
 
 
@@ -232,18 +240,23 @@ def check_tunnel(tunnel_name):
             result = 'up' if proc_started(get_supervisor().getProcessInfo(tunnel_name)) else 'down'
         elif ctype == 'url':
             rsp = requests.head(chk['url'], verify=False, timeout=timeout)
-            log.debug('{} check ok: {} {}'.format(tunnel_name, chk, rsp))
+            ok = rsp.status_code >= 200 and rsp.status_code < 300
+            log.debug('{} check {}: {} {}'.format(
+                tunnel_name,
+                'ok' if ok else "failed",
+                chk,
+                rsp))
             result = 'up' if rsp.status_code >= 200 and rsp.status_code < 300 else 'down'
         elif ctype == 'db':
             import sqlalchemy
-            eng = get_run_data(tunnel_name)['db_engine'];
+            eng = get_run_data(tunnel_name).get('db_engine');
             if not eng:
                 eng = dbengine_create_func(chk)
                 get_run_data(tunnel_name)['db_engine'] = eng
             conn = eng.connect()
             conn.close()
             log.debug('{} check ok: {}'.format(tunnel_name, chk))
-            result = 'up'
+            result =     'up'
         elif ctype == 'socket':
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             s.settimeout(timeout)
@@ -254,6 +267,7 @@ def check_tunnel(tunnel_name):
         else:
             result = 'invalid'
     except Exception as e:
+        log.error(str(e))
         log.debug('{} check failed: {} {}'.format(tunnel_name, chk, e))
         result = 'down'
 
@@ -296,21 +310,22 @@ def stop_dependent_tunnels(tunnel_name):
             stop_dependent_tunnels(tunnel)
 
 
-def check_dependent_tunnels(check_result_parent = None, check_only=False):
+def check_dependent_tunnels(check_result_parent=None, check_only=False):
 
     tunnels = get_tunnel_dependents(
         check_result_parent.name if check_result_parent else None);
 
     if not tunnels:
-        # log.debug('no dependents\n')
+        # log.debug('check_dependent_tunnels: no dependents\n')
         return {}
 
+    log.debug('check_dependent_tunnels: ' + ','.join(tunnels))
 
     statuses = []
-    if check_result_parent and not check_result_parent.status == 'up':
+    if check_result_parent and check_result_parent.status != 'up':
         # If parent is not up, then skip children
-        statuses = [CheckResult(tunnel_name,'skipped',get_check_type(get_check_cfg(tunnel_name)))
-            for tunnel_name in tunnels ]
+        statuses = [CheckResult(tunnel_name, 'skipped', get_check_type(get_check_cfg(tunnel_name)))
+                    for tunnel_name in tunnels]
     else:
         # Use a thread pool to perform the checks of child tunnels
         with futures.ThreadPoolExecutor(max_workers=10) as executor:
@@ -329,6 +344,7 @@ def check_dependent_tunnels(check_result_parent = None, check_only=False):
         if check_result.status != 'down' or check_only:
             statuses.extend(check_dependent_tunnels(check_result, check_only))
         else:
+            pass
             # Make sure dependent tunnels are stopped. Don't bother gathering status as we may no longer use it
             stop_dependent_tunnels(check_result.name)
 
@@ -360,7 +376,7 @@ def vpnmonitor():
             results = check_dependent_tunnels()
             # for result in results:
             #     log.debug('{} check: {}'.format(result.name, result.status))
-            time.sleep(6)
+            time.sleep(20)
             # childutils.listener.ok()
         except Exception as e:
             print e
@@ -468,7 +484,7 @@ def get_tunnel_dependents(tunnel_name=None):
 
 
 def get_run_data(tunnel_name):
-    data = get_config()['tunnels'][tunnel_name]['__run_data'];
+    data = get_config()['tunnels'][tunnel_name].get('__run_data');
     if not data:
         data = {}
         get_config()['tunnels'][tunnel_name]['__run_data'] = data

requirements.txt

@@ -5,4 +5,4 @@ supervisor==3.3.1
 requests==2.10.0
 Jinja2==2.8
 futures==3.0.5
-sshuttle===0.78.3
+sshuttle===0.78.1

setup.py

@@ -1,21 +1,17 @@
 # https://packaging.python.org/tutorials/distributing-packages/
 
 from setuptools import setup
-# import pypandoc
 
-
-# readme = open('README_GENERATED.rst', 'r')
-# README_TEXT = pypandoc.convert_file('README.md', 'rst')
-# readme.close()
-
-import os
-readme = open('README.md', 'r')
-README_TEXT = readme.read()
-readme.close()
+try:
+    readme = open('README.rst', 'r')
+    README_TEXT = readme.read()
+    readme.close()
+except:
+    README_TEXT = ""
 
 setup(
     name='no_you_talk_to_the_hand',
-    version='1.0.3',
+    version='1.0.4',
     author='flashashen',
     author_email='flashashen@gmail.com',
     description='nyttth: Boss your corporate network effortlessly. Automatic and organized tunneling with sshuttle + supervisord + yaml',
@@ -33,7 +29,7 @@
 
     ],
     platforms='osx,linux',
-    keywords = "ssh vpn tunnel forward daemon",
+    keywords="ssh vpn tunnel forward daemon",
     long_description=README_TEXT,
     py_modules=['no_you_talk_to_the_hand'],
     install_requires=[
@@ -44,7 +40,7 @@
         'six',
         'requests',
         'futures',
-        'sshuttle==0.78.4' 
+        'sshuttle==0.78.1'
     ],
 
     entry_points='''