From 19a74cd7b09654e0534a174c34a213da6f84fd63 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 2 Jun 2019 18:06:17 +0200 Subject: [PATCH 1/3] PyMySQL Instrumentation, support and tests --- instana/__init__.py | 1 + instana/instrumentation/pymysql.py | 17 +++ setup.py | 9 +- tests/helpers.py | 7 +- tests/test_mysql-python.py | 1 - tests/test_pymysql.py | 216 +++++++++++++++++++++++++++++ 6 files changed, 241 insertions(+), 10 deletions(-) create mode 100644 instana/instrumentation/pymysql.py create mode 100644 tests/test_pymysql.py diff --git a/instana/__init__.py b/instana/__init__.py index 810e7671..f6d143c3 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -65,6 +65,7 @@ def boot_agent(): from .instrumentation.tornado import server from .instrumentation import logging from .instrumentation import mysqlpython + from .instrumentation import pymysql from .instrumentation import redis from .instrumentation import sqlalchemy from .instrumentation import sudsjurko diff --git a/instana/instrumentation/pymysql.py b/instana/instrumentation/pymysql.py new file mode 100644 index 00000000..59811c60 --- /dev/null +++ b/instana/instrumentation/pymysql.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +from ..log import logger +from .pep0249 import ConnectionFactory + +try: + import pymysql # + + cf = ConnectionFactory(connect_func=pymysql.connect, module_name='mysql') + + setattr(pymysql, 'connect', cf) + if hasattr(pymysql, 'Connect'): + setattr(pymysql, 'Connect', cf) + + logger.debug("Instrumenting pymysql") +except ImportError: + pass diff --git a/setup.py b/setup.py index 6d27b39c..2a32a1b2 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ # coding: utf-8 -from distutils.version import LooseVersion -from setuptools import find_packages, setup import sys from os import path +from distutils.version import LooseVersion +from setuptools import find_packages, setup # Import README.md into long_description pwd = path.abspath(path.dirname(__file__)) @@ -16,6 +16,7 @@ def check_setuptools(): + """ Validate that we have min version required of setuptools """ import pkg_resources st_version = pkg_resources.get_distribution('setuptools').version if LooseVersion(st_version) < LooseVersion('20.2.2'): @@ -73,6 +74,7 @@ def check_setuptools(): 'mock>=2.0.0', 'MySQL-python>=1.2.5;python_version<="2.7"', 'psycopg2>=2.7.1', + 'PyMySQL[rsa]>=0.9.1', 'pyOpenSSL>=16.1.0;python_version<="2.7"', 'pytest>=3.0.1', 'redis<3.0.0', @@ -85,7 +87,8 @@ def check_setuptools(): ], }, test_suite='nose.collector', - keywords=['performance', 'opentracing', 'metrics', 'monitoring', 'tracing', 'distributed-tracing'], + keywords=['performance', 'opentracing', 'metrics', 'monitoring', + 'tracing', 'distributed-tracing'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Framework :: Django', diff --git a/tests/helpers.py b/tests/helpers.py index 8d8b3c16..96aec224 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -15,13 +15,8 @@ testenv['mysql_port'] = int(os.environ.get('MYSQL_PORT', '3306')) testenv['mysql_db'] = os.environ.get('MYSQL_DB', 'circle_test') testenv['mysql_user'] = os.environ.get('MYSQL_USER', 'root') +testenv['mysql_pw'] = os.environ.get('MYSQL_PW', '') -if 'MYSQL_PW' in os.environ: - testenv['mysql_pw'] = os.environ['MYSQL_PW'] -elif 'TRAVIS_MYSQL_PASS' in os.environ: - testenv['mysql_pw'] = os.environ['TRAVIS_MYSQL_PASS'] -else: - testenv['mysql_pw'] = '' """ PostgreSQL Environment diff --git a/tests/test_mysql-python.py b/tests/test_mysql-python.py index b0b03f5b..4cd7e203 100644 --- a/tests/test_mysql-python.py +++ b/tests/test_mysql-python.py @@ -1,7 +1,6 @@ from __future__ import absolute_import import logging -import os import sys from unittest import SkipTest diff --git a/tests/test_pymysql.py b/tests/test_pymysql.py new file mode 100644 index 00000000..e47ecfc9 --- /dev/null +++ b/tests/test_pymysql.py @@ -0,0 +1,216 @@ +from __future__ import absolute_import + +import logging +import sys +from unittest import SkipTest + +from nose.tools import assert_equals + +from instana.singletons import tracer + +from .helpers import testenv + +import pymysql + +logger = logging.getLogger(__name__) + +create_table_query = 'CREATE TABLE IF NOT EXISTS users(id serial primary key, \ + name varchar(40) NOT NULL, email varchar(40) NOT NULL)' + +create_proc_query = """ +CREATE PROCEDURE test_proc(IN t VARCHAR(255)) +BEGIN + SELECT name FROM users WHERE name = t; +END +""" + +db = pymysql.connect(host=testenv['mysql_host'], port=testenv['mysql_port'], + user=testenv['mysql_user'], passwd=testenv['mysql_pw'], + db=testenv['mysql_db']) + +cursor = db.cursor() +cursor.execute(create_table_query) + +while cursor.nextset() is not None: + pass + +cursor.execute('DROP PROCEDURE IF EXISTS test_proc') + +while cursor.nextset() is not None: + pass + +cursor.execute(create_proc_query) + +while cursor.nextset() is not None: + pass + +cursor.close() +db.close() + + +class TestPyMySQL: + def setUp(self): + logger.warn("MySQL connecting: %s:@%s:3306/%s", testenv['mysql_user'], testenv['mysql_host'], testenv['mysql_db']) + self.db = pymysql.connect(host=testenv['mysql_host'], port=testenv['mysql_port'], + user=testenv['mysql_user'], passwd=testenv['mysql_pw'], + db=testenv['mysql_db']) + self.cursor = self.db.cursor() + self.recorder = tracer.recorder + self.recorder.clear_spans() + tracer.cur_ctx = None + + def tearDown(self): + """ Do nothing for now """ + return None + + def test_vanilla_query(self): + self.cursor.execute("""SELECT * from users""") + result = self.cursor.fetchone() + assert_equals(3, len(result)) + + spans = self.recorder.queued_spans() + assert_equals(0, len(spans)) + + def test_basic_query(self): + result = None + with tracer.start_active_span('test'): + result = self.cursor.execute("""SELECT * from users""") + self.cursor.fetchone() + + assert(result >= 0) + + spans = self.recorder.queued_spans() + assert_equals(2, len(spans)) + + db_span = spans[0] + test_span = spans[1] + + assert_equals("test", test_span.data.sdk.name) + assert_equals(test_span.t, db_span.t) + assert_equals(db_span.p, test_span.s) + + assert_equals(None, db_span.error) + assert_equals(None, db_span.ec) + + assert_equals(db_span.n, "mysql") + assert_equals(db_span.data.mysql.db, testenv['mysql_db']) + assert_equals(db_span.data.mysql.user, testenv['mysql_user']) + assert_equals(db_span.data.mysql.stmt, 'SELECT * from users') + assert_equals(db_span.data.mysql.host, "%s:3306" % testenv['mysql_host']) + + def test_basic_insert(self): + result = None + with tracer.start_active_span('test'): + result = self.cursor.execute( + """INSERT INTO users(name, email) VALUES(%s, %s)""", + ('beaker', 'beaker@muppets.com')) + + assert_equals(1, result) + + spans = self.recorder.queued_spans() + assert_equals(2, len(spans)) + + db_span = spans[0] + test_span = spans[1] + + assert_equals("test", test_span.data.sdk.name) + assert_equals(test_span.t, db_span.t) + assert_equals(db_span.p, test_span.s) + + assert_equals(None, db_span.error) + assert_equals(None, db_span.ec) + + assert_equals(db_span.n, "mysql") + assert_equals(db_span.data.mysql.db, testenv['mysql_db']) + assert_equals(db_span.data.mysql.user, testenv['mysql_user']) + assert_equals(db_span.data.mysql.stmt, 'INSERT INTO users(name, email) VALUES(%s, %s)') + assert_equals(db_span.data.mysql.host, "%s:3306" % testenv['mysql_host']) + + def test_executemany(self): + result = None + with tracer.start_active_span('test'): + result = self.cursor.executemany("INSERT INTO users(name, email) VALUES(%s, %s)", + [('beaker', 'beaker@muppets.com'), ('beaker', 'beaker@muppets.com')]) + self.db.commit() + + assert_equals(2, result) + + spans = self.recorder.queued_spans() + assert_equals(2, len(spans)) + + db_span = spans[0] + test_span = spans[1] + + assert_equals("test", test_span.data.sdk.name) + assert_equals(test_span.t, db_span.t) + assert_equals(db_span.p, test_span.s) + + assert_equals(None, db_span.error) + assert_equals(None, db_span.ec) + + assert_equals(db_span.n, "mysql") + assert_equals(db_span.data.mysql.db, testenv['mysql_db']) + assert_equals(db_span.data.mysql.user, testenv['mysql_user']) + assert_equals(db_span.data.mysql.stmt, 'INSERT INTO users(name, email) VALUES(%s, %s)') + assert_equals(db_span.data.mysql.host, "%s:3306" % testenv['mysql_host']) + + def test_call_proc(self): + result = None + with tracer.start_active_span('test'): + result = self.cursor.callproc('test_proc', ('beaker',)) + + assert(result) + + spans = self.recorder.queued_spans() + assert_equals(2, len(spans)) + + db_span = spans[0] + test_span = spans[1] + + assert_equals("test", test_span.data.sdk.name) + assert_equals(test_span.t, db_span.t) + assert_equals(db_span.p, test_span.s) + + assert_equals(None, db_span.error) + assert_equals(None, db_span.ec) + + assert_equals(db_span.n, "mysql") + assert_equals(db_span.data.mysql.db, testenv['mysql_db']) + assert_equals(db_span.data.mysql.user, testenv['mysql_user']) + assert_equals(db_span.data.mysql.stmt, 'test_proc') + assert_equals(db_span.data.mysql.host, "%s:3306" % testenv['mysql_host']) + + def test_error_capture(self): + result = None + span = None + try: + with tracer.start_active_span('test'): + result = self.cursor.execute("""SELECT * from blah""") + self.cursor.fetchone() + except Exception: + pass + finally: + if span: + span.finish() + + assert(result is None) + + spans = self.recorder.queued_spans() + assert_equals(2, len(spans)) + + db_span = spans[0] + test_span = spans[1] + + assert_equals("test", test_span.data.sdk.name) + assert_equals(test_span.t, db_span.t) + assert_equals(db_span.p, test_span.s) + + assert_equals(True, db_span.error) + assert_equals(1, db_span.ec) + assert_equals(db_span.data.mysql.error, '(1146, "Table \'%s.blah\' doesn\'t exist")' % testenv['mysql_db']) + + assert_equals(db_span.n, "mysql") + assert_equals(db_span.data.mysql.db, testenv['mysql_db']) + assert_equals(db_span.data.mysql.user, testenv['mysql_user']) + assert_equals(db_span.data.mysql.stmt, 'SELECT * from blah') + assert_equals(db_span.data.mysql.host, "%s:3306" % testenv['mysql_host']) From 37f940c44d312427c081b2a5cd4c1f9744416b91 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 2 Jun 2019 18:22:12 +0200 Subject: [PATCH 2/3] Add py2 special case --- tests/test_pymysql.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_pymysql.py b/tests/test_pymysql.py index e47ecfc9..27540510 100644 --- a/tests/test_pymysql.py +++ b/tests/test_pymysql.py @@ -207,7 +207,13 @@ def test_error_capture(self): assert_equals(True, db_span.error) assert_equals(1, db_span.ec) - assert_equals(db_span.data.mysql.error, '(1146, "Table \'%s.blah\' doesn\'t exist")' % testenv['mysql_db']) + + if sys.version_info[0] >= 3: + # Python 3 + assert_equals(db_span.data.mysql.error, u'(1146, "Table \'%s.blah\' doesn\'t exist")' % testenv['mysql_db']) + else: + # Python 2 + assert_equals(db_span.data.mysql.error, u'(1146, u"Table \'%s.blah\' doesn\'t exist")' % testenv['mysql_db']) assert_equals(db_span.n, "mysql") assert_equals(db_span.data.mysql.db, testenv['mysql_db']) From 067bafb53f305b9a488c6240da32432428b3b686 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 4 Jun 2019 16:13:41 +0200 Subject: [PATCH 3/3] Add sql_sanitizer and sanitize queries --- instana/instrumentation/pep0249.py | 3 ++- instana/util.py | 21 +++++++++++++++++++-- tests/test_pymysql.py | 27 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/instana/instrumentation/pep0249.py b/instana/instrumentation/pep0249.py index 95df29f1..b58cac91 100644 --- a/instana/instrumentation/pep0249.py +++ b/instana/instrumentation/pep0249.py @@ -4,6 +4,7 @@ from ..log import logger from ..singletons import tracer +from ..util import sql_sanitizer class CursorWrapper(wrapt.ObjectProxy): @@ -20,7 +21,7 @@ def _collect_kvs(self, span, sql): try: span.set_tag(ext.SPAN_KIND, 'exit') span.set_tag(ext.DATABASE_INSTANCE, self._connect_params[1]['db']) - span.set_tag(ext.DATABASE_STATEMENT, sql) + span.set_tag(ext.DATABASE_STATEMENT, sql_sanitizer(sql)) span.set_tag(ext.DATABASE_TYPE, 'mysql') span.set_tag(ext.DATABASE_USER, self._connect_params[1]['user']) span.set_tag('host', "%s:%s" % diff --git a/instana/util.py b/instana/util.py index 1464f7e6..f205b5bb 100644 --- a/instana/util.py +++ b/instana/util.py @@ -182,6 +182,20 @@ def strip_secrets(qp, matcher, kwlist): logger.debug("strip_secrets", exc_info=True) +def sql_sanitizer(sql): + """ + Removes values from valid SQL statements and returns a stripped version. + + :param sql: The SQL statement to be sanitized + :return: String - A sanitized SQL statement without values. + """ + return regexp_sql_values.sub('?', sql) + + +# Used by sql_sanitizer +regexp_sql_values = re.compile('(\'[\s\S][^\']*\'|\d*\.\d+|\d+|NULL)') + + def get_default_gateway(): """ Attempts to read /proc/self/net/route to determine the default gateway in use. @@ -230,6 +244,9 @@ def get_py_source(file): finally: return response +# Used by get_py_source +regexp_py = re.compile('\.py$') + def every(delay, task, name): """ @@ -253,5 +270,5 @@ def every(delay, task, name): next_time += (time.time() - next_time) // delay * delay + delay -# Used by get_py_source -regexp_py = re.compile('\.py$') + + diff --git a/tests/test_pymysql.py b/tests/test_pymysql.py index 27540510..cc3989f5 100644 --- a/tests/test_pymysql.py +++ b/tests/test_pymysql.py @@ -98,6 +98,33 @@ def test_basic_query(self): assert_equals(db_span.data.mysql.stmt, 'SELECT * from users') assert_equals(db_span.data.mysql.host, "%s:3306" % testenv['mysql_host']) + def test_query_with_params(self): + result = None + with tracer.start_active_span('test'): + result = self.cursor.execute("""SELECT * from users where id=1""") + self.cursor.fetchone() + + assert(result >= 0) + + spans = self.recorder.queued_spans() + assert_equals(2, len(spans)) + + db_span = spans[0] + test_span = spans[1] + + assert_equals("test", test_span.data.sdk.name) + assert_equals(test_span.t, db_span.t) + assert_equals(db_span.p, test_span.s) + + assert_equals(None, db_span.error) + assert_equals(None, db_span.ec) + + assert_equals(db_span.n, "mysql") + assert_equals(db_span.data.mysql.db, testenv['mysql_db']) + assert_equals(db_span.data.mysql.user, testenv['mysql_user']) + assert_equals(db_span.data.mysql.stmt, 'SELECT * from users where id=?') + assert_equals(db_span.data.mysql.host, "%s:3306" % testenv['mysql_host']) + def test_basic_insert(self): result = None with tracer.start_active_span('test'):