From 0b7adb5a85ac77516652f609ff6bb640785962fe Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 12 Nov 2018 15:28:41 +0100 Subject: [PATCH 1/3] Initial Redis instrumentation, tests and infra --- instana/__init__.py | 1 + instana/instrumentation/redis.py | 44 ++++++++++++++++++++ instana/json_span.py | 12 ++++++ instana/recorder.py | 17 +++++--- setup.py | 1 + tests/helpers.py | 8 ++++ tests/test_redis.py | 70 ++++++++++++++++++++++++++++++++ 7 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 instana/instrumentation/redis.py create mode 100644 tests/test_redis.py diff --git a/instana/__init__.py b/instana/__init__.py index 335b345c..bc5e34bc 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -61,6 +61,7 @@ def load_instrumentation(): # Import & initialize instrumentation from .instrumentation import asynqp # noqa from .instrumentation import mysqlpython # noqa + from .instrumentation import redis # noqa from .instrumentation import sqlalchemy # noqa from .instrumentation import sudsjurko # noqa from .instrumentation import urllib3 # noqa diff --git a/instana/instrumentation/redis.py b/instana/instrumentation/redis.py new file mode 100644 index 00000000..f5226748 --- /dev/null +++ b/instana/instrumentation/redis.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import + +import opentracing +import opentracing.ext.tags as ext +import wrapt + +from ..log import logger +from ..singletons import tracer + +try: + import redis + + @wrapt.patch_function_wrapper('redis.client','StrictRedis.execute_command') + def execute_command_with_instana(wrapped, instance, args, kwargs): + parent_span = tracer.active_span + + # import ipdb; ipdb.set_trace() + + # If we're not tracing, just return + if parent_span is None: + return wrapped(*args, **kwargs) + + with tracer.start_active_span("redis", child_of=parent_span) as scope: + + try: + ckw = instance.connection_pool.connection_kwargs + url = "redis://%s:%d/%d" % (ckw['host'], ckw['port'], ckw['db']) + scope.span.set_tag("connection", url) + scope.span.set_tag("driver", "redis-py") + scope.span.set_tag("command", args[0]) + + rv = wrapped(*args, **kwargs) + except Exception as e: + scope.span.log_kv({'message': e}) + scope.span.set_tag("error", True) + ec = scope.span.tags.get('ec', 0) + scope.span.set_tag("ec", ec+1) + raise + else: + return rv + + logger.debug("Instrumenting redis") +except ImportError: + pass diff --git a/instana/json_span.py b/instana/json_span.py index 74a6dc68..f46508b6 100644 --- a/instana/json_span.py +++ b/instana/json_span.py @@ -30,6 +30,7 @@ class Data(object): custom = None http = None rabbitmq = None + redis = None sdk = None service = None sqlalchemy = None @@ -72,6 +73,17 @@ class RabbitmqData(object): def __init__(self, **kwds): self.__dict__.update(kwds) + +class RedisData(object): + connection = None + driver = None + command = None + error = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) + + class SQLAlchemyData(object): sql = None url = None diff --git a/instana/recorder.py b/instana/recorder.py index bb00062a..65b6a8b2 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -12,7 +12,8 @@ import instana.singletons from .json_span import (CustomData, Data, HttpData, JsonSpan, MySQLData, - RabbitmqData, SDKData, SoapData, SQLAlchemyData) + RabbitmqData, RedisData, SDKData, SoapData, + SQLAlchemyData) from .log import logger if sys.version_info.major is 2: @@ -22,12 +23,12 @@ class InstanaRecorder(SpanRecorder): - registered_spans = ("django", "memcache", "mysql", "rabbitmq", "rpc-client", - "rpc-server", "sqlalchemy", "soap", "urllib3", "wsgi") + registered_spans = ("django", "memcache", "mysql", "rabbitmq", "redis", + "rpc-client", "rpc-server", "sqlalchemy", "soap", "urllib3", "wsgi") http_spans = ("django", "wsgi", "urllib3", "soap") - exit_spans = ("memcache", "mysql", "rabbitmq", "rpc-client", "sqlalchemy", - "soap", "urllib3") + exit_spans = ("memcache", "mysql", "rabbitmq", "redis", "rpc-client", + "sqlalchemy", "soap", "urllib3") entry_spans = ("django", "wsgi", "rabbitmq", "rpc-server") entry_kind = ["entry", "server", "consumer"] @@ -115,6 +116,12 @@ def build_registered_span(self, span): address=span.tags.pop('address', None), key=span.tags.pop('key', None)) + if span.operation_name == "redis": + data.redis = RedisData(connection=span.tags.pop('connection', None), + driver=span.tags.pop('driver', None), + command=span.tags.pop('command', None), + error=span.tags.pop('error', None)) + if span.operation_name == "sqlalchemy": data.sqlalchemy = SQLAlchemyData(sql=span.tags.pop('sqlalchemy.sql', None), eng=span.tags.pop('sqlalchemy.eng', None), diff --git a/setup.py b/setup.py index efaeab90..824850d0 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ def check_setuptools(): 'psycopg2>=2.7.1', 'pyOpenSSL>=16.1.0;python_version<="2.7"', 'pytest>=3.0.1', + 'redis>=2.10.6', 'requests>=2.17.1', 'sqlalchemy>=1.1.15', 'spyne>=2.9', diff --git a/tests/helpers.py b/tests/helpers.py index 59aa7cb7..15713791 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -43,3 +43,11 @@ testenv['postgresql_pw'] = os.environ['TRAVIS_POSTGRESQL_PASS'] else: testenv['postgresql_pw'] = '' + +""" +Redis Environment +""" +if 'REDIS' in os.environ: + testenv['redis_url']= os.environ['REDIS'] +else: + testenv['redis_url'] = '127.0.0.1:6379' diff --git a/tests/test_redis.py b/tests/test_redis.py new file mode 100644 index 00000000..c0954d38 --- /dev/null +++ b/tests/test_redis.py @@ -0,0 +1,70 @@ +from __future__ import absolute_import + +import os +import sys +import unittest + +import redis + +from .helpers import testenv +from instana.singletons import tracer + + +class TestRedis(unittest.TestCase): + def setUp(self): + """ Clear all spans before a test run """ + self.recorder = tracer.recorder + self.recorder.clear_spans() + self.rc = redis.StrictRedis.from_url("redis://%s/0" % testenv['redis_url']) + + def tearDown(self): + pass + + def test_set_get(self): + result = None + with tracer.start_active_span('test'): + self.rc.set('foox', 'barX') + self.rc.set('fooy', 'barY') + result = self.rc.get('foox') + + spans = self.recorder.queued_spans() + self.assertEqual(4, len(spans)) + + rs1_span = spans[0] + rs2_span = spans[1] + rs3_span = spans[2] + test_span = spans[3] + + self.assertIsNone(tracer.active_span) + + # Same traceId + self.assertEqual(test_span.t, rs1_span.t) + self.assertEqual(test_span.t, rs2_span.t) + self.assertEqual(test_span.t, rs3_span.t) + + # Parent relationships + self.assertEqual(rs1_span.p, test_span.s) + self.assertEqual(rs2_span.p, test_span.s) + self.assertEqual(rs3_span.p, test_span.s) + + # Error logging + self.assertFalse(test_span.error) + self.assertIsNone(test_span.ec) + self.assertFalse(rs1_span.error) + self.assertIsNone(rs1_span.ec) + self.assertFalse(rs2_span.error) + self.assertIsNone(rs2_span.ec) + self.assertFalse(rs3_span.error) + self.assertIsNone(rs3_span.ec) + + # Redis span + self.assertEqual('redis', rs1_span.n) + self.assertFalse('custom' in rs1_span.data.__dict__) + self.assertTrue('redis' in rs1_span.data.__dict__) + + self.assertEqual('redis-py', rs1_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs1_span.data.redis.connection) + + self.assertIsNotNone(rs1_span.stack) + self.assertTrue(type(rs1_span.stack) is list) + self.assertGreater(len(rs1_span.stack), 0) From 34def5f19cee300ce375c562c6ffc5795694f5d9 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 12 Nov 2018 17:28:37 +0100 Subject: [PATCH 2/3] Pipeline support; subCommands; More Tests --- instana/instrumentation/redis.py | 39 ++++- instana/json_span.py | 1 + instana/recorder.py | 3 +- tests/test_redis.py | 251 ++++++++++++++++++++++++++++++- 4 files changed, 286 insertions(+), 8 deletions(-) diff --git a/instana/instrumentation/redis.py b/instana/instrumentation/redis.py index f5226748..00082534 100644 --- a/instana/instrumentation/redis.py +++ b/instana/instrumentation/redis.py @@ -14,8 +14,6 @@ def execute_command_with_instana(wrapped, instance, args, kwargs): parent_span = tracer.active_span - # import ipdb; ipdb.set_trace() - # If we're not tracing, just return if parent_span is None: return wrapped(*args, **kwargs) @@ -39,6 +37,43 @@ def execute_command_with_instana(wrapped, instance, args, kwargs): else: return rv + @wrapt.patch_function_wrapper('redis.client','BasePipeline.execute') + def execute_with_instana(wrapped, instance, args, kwargs): + parent_span = tracer.active_span + + # If we're not tracing, just return + if parent_span is None: + return wrapped(*args, **kwargs) + + with tracer.start_active_span("redis", child_of=parent_span) as scope: + + try: + ckw = instance.connection_pool.connection_kwargs + url = "redis://%s:%d/%d" % (ckw['host'], ckw['port'], ckw['db']) + scope.span.set_tag("connection", url) + scope.span.set_tag("driver", "redis-py") + scope.span.set_tag("command", 'PIPELINE') + + try: + pipe_cmds = [] + for e in instance.command_stack: + pipe_cmds.append(e[0][0]) + scope.span.set_tag("subCommands", pipe_cmds) + except Exception as e: + # If anything breaks during cmd collection, just log a + # debug message + logger.debug("Error collecting pipeline commands") + + rv = wrapped(*args, **kwargs) + except Exception as e: + scope.span.log_kv({'message': e}) + scope.span.set_tag("error", True) + ec = scope.span.tags.get('ec', 0) + scope.span.set_tag("ec", ec+1) + raise + else: + return rv + logger.debug("Instrumenting redis") except ImportError: pass diff --git a/instana/json_span.py b/instana/json_span.py index f46508b6..522b1248 100644 --- a/instana/json_span.py +++ b/instana/json_span.py @@ -79,6 +79,7 @@ class RedisData(object): driver = None command = None error = None + subCommands = None def __init__(self, **kwds): self.__dict__.update(kwds) diff --git a/instana/recorder.py b/instana/recorder.py index 65b6a8b2..774e9e4d 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -120,7 +120,8 @@ def build_registered_span(self, span): data.redis = RedisData(connection=span.tags.pop('connection', None), driver=span.tags.pop('driver', None), command=span.tags.pop('command', None), - error=span.tags.pop('error', None)) + error=span.tags.pop('error', None), + subCommands=span.tags.pop('subCommands', None)) if span.operation_name == "sqlalchemy": data.sqlalchemy = SQLAlchemyData(sql=span.tags.pop('sqlalchemy.sql', None), diff --git a/tests/test_redis.py b/tests/test_redis.py index c0954d38..442f70eb 100644 --- a/tests/test_redis.py +++ b/tests/test_redis.py @@ -15,7 +15,8 @@ def setUp(self): """ Clear all spans before a test run """ self.recorder = tracer.recorder self.recorder.clear_spans() - self.rc = redis.StrictRedis.from_url("redis://%s/0" % testenv['redis_url']) + self.strict_redis = redis.StrictRedis.from_url("redis://%s/0" % testenv['redis_url']) + self.redis = redis.Redis.from_url("redis://%s/0" % testenv['redis_url']) def tearDown(self): pass @@ -23,13 +24,177 @@ def tearDown(self): def test_set_get(self): result = None with tracer.start_active_span('test'): - self.rc.set('foox', 'barX') - self.rc.set('fooy', 'barY') - result = self.rc.get('foox') + self.strict_redis.set('foox', 'barX') + self.strict_redis.set('fooy', 'barY') + result = self.strict_redis.get('foox') spans = self.recorder.queued_spans() self.assertEqual(4, len(spans)) + self.assertEqual(b'barX', result) + + rs1_span = spans[0] + rs2_span = spans[1] + rs3_span = spans[2] + test_span = spans[3] + + self.assertIsNone(tracer.active_span) + + # Same traceId + self.assertEqual(test_span.t, rs1_span.t) + self.assertEqual(test_span.t, rs2_span.t) + self.assertEqual(test_span.t, rs3_span.t) + + # Parent relationships + self.assertEqual(rs1_span.p, test_span.s) + self.assertEqual(rs2_span.p, test_span.s) + self.assertEqual(rs3_span.p, test_span.s) + + # Error logging + self.assertFalse(test_span.error) + self.assertIsNone(test_span.ec) + self.assertFalse(rs1_span.error) + self.assertIsNone(rs1_span.ec) + self.assertFalse(rs2_span.error) + self.assertIsNone(rs2_span.ec) + self.assertFalse(rs3_span.error) + self.assertIsNone(rs3_span.ec) + + # Redis span 1 + self.assertEqual('redis', rs1_span.n) + self.assertFalse('custom' in rs1_span.data.__dict__) + self.assertTrue('redis' in rs1_span.data.__dict__) + + self.assertEqual('redis-py', rs1_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs1_span.data.redis.connection) + self.assertEqual("SET", rs1_span.data.redis.command) + self.assertIsNone(rs1_span.data.redis.error) + + self.assertIsNotNone(rs1_span.stack) + self.assertTrue(type(rs1_span.stack) is list) + self.assertGreater(len(rs1_span.stack), 0) + + # Redis span 2 + self.assertEqual('redis', rs2_span.n) + self.assertFalse('custom' in rs2_span.data.__dict__) + self.assertTrue('redis' in rs2_span.data.__dict__) + + self.assertEqual('redis-py', rs2_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs2_span.data.redis.connection) + self.assertEqual("SET", rs2_span.data.redis.command) + self.assertIsNone(rs2_span.data.redis.error) + + self.assertIsNotNone(rs2_span.stack) + self.assertTrue(type(rs2_span.stack) is list) + self.assertGreater(len(rs2_span.stack), 0) + + # Redis span 3 + self.assertEqual('redis', rs3_span.n) + self.assertFalse('custom' in rs3_span.data.__dict__) + self.assertTrue('redis' in rs3_span.data.__dict__) + + self.assertEqual('redis-py', rs3_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs3_span.data.redis.connection) + self.assertEqual("GET", rs3_span.data.redis.command) + self.assertIsNone(rs3_span.data.redis.error) + + self.assertIsNotNone(rs3_span.stack) + self.assertTrue(type(rs3_span.stack) is list) + self.assertGreater(len(rs3_span.stack), 0) + + def test_set_incr_get(self): + result = None + with tracer.start_active_span('test'): + self.strict_redis.set('counter', '10') + self.strict_redis.incr('counter') + result = self.strict_redis.get('counter') + + spans = self.recorder.queued_spans() + self.assertEqual(4, len(spans)) + + self.assertEqual(b'11', result) + + rs1_span = spans[0] + rs2_span = spans[1] + rs3_span = spans[2] + test_span = spans[3] + + self.assertIsNone(tracer.active_span) + + # Same traceId + self.assertEqual(test_span.t, rs1_span.t) + self.assertEqual(test_span.t, rs2_span.t) + self.assertEqual(test_span.t, rs3_span.t) + + # Parent relationships + self.assertEqual(rs1_span.p, test_span.s) + self.assertEqual(rs2_span.p, test_span.s) + self.assertEqual(rs3_span.p, test_span.s) + + # Error logging + self.assertFalse(test_span.error) + self.assertIsNone(test_span.ec) + self.assertFalse(rs1_span.error) + self.assertIsNone(rs1_span.ec) + self.assertFalse(rs2_span.error) + self.assertIsNone(rs2_span.ec) + self.assertFalse(rs3_span.error) + self.assertIsNone(rs3_span.ec) + + # Redis span 1 + self.assertEqual('redis', rs1_span.n) + self.assertFalse('custom' in rs1_span.data.__dict__) + self.assertTrue('redis' in rs1_span.data.__dict__) + + self.assertEqual('redis-py', rs1_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs1_span.data.redis.connection) + self.assertEqual("SET", rs1_span.data.redis.command) + self.assertIsNone(rs1_span.data.redis.error) + + self.assertIsNotNone(rs1_span.stack) + self.assertTrue(type(rs1_span.stack) is list) + self.assertGreater(len(rs1_span.stack), 0) + + # Redis span 2 + self.assertEqual('redis', rs2_span.n) + self.assertFalse('custom' in rs2_span.data.__dict__) + self.assertTrue('redis' in rs2_span.data.__dict__) + + self.assertEqual('redis-py', rs2_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs2_span.data.redis.connection) + self.assertEqual("INCRBY", rs2_span.data.redis.command) + self.assertIsNone(rs2_span.data.redis.error) + + self.assertIsNotNone(rs2_span.stack) + self.assertTrue(type(rs2_span.stack) is list) + self.assertGreater(len(rs2_span.stack), 0) + + # Redis span 3 + self.assertEqual('redis', rs3_span.n) + self.assertFalse('custom' in rs3_span.data.__dict__) + self.assertTrue('redis' in rs3_span.data.__dict__) + + self.assertEqual('redis-py', rs3_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs3_span.data.redis.connection) + self.assertEqual("GET", rs3_span.data.redis.command) + self.assertIsNone(rs3_span.data.redis.error) + + self.assertIsNotNone(rs3_span.stack) + self.assertTrue(type(rs3_span.stack) is list) + self.assertGreater(len(rs3_span.stack), 0) + + def test_old_redis_client(self): + result = None + with tracer.start_active_span('test'): + self.redis.set('foox', 'barX') + self.redis.set('fooy', 'barY') + result = self.redis.get('foox') + + spans = self.recorder.queued_spans() + self.assertEqual(4, len(spans)) + + self.assertEqual(b'barX', result) + rs1_span = spans[0] rs2_span = spans[1] rs3_span = spans[2] @@ -57,13 +222,89 @@ def test_set_get(self): self.assertFalse(rs3_span.error) self.assertIsNone(rs3_span.ec) - # Redis span + # Redis span 1 + self.assertEqual('redis', rs1_span.n) + self.assertFalse('custom' in rs1_span.data.__dict__) + self.assertTrue('redis' in rs1_span.data.__dict__) + + self.assertEqual('redis-py', rs1_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs1_span.data.redis.connection) + self.assertEqual("SET", rs1_span.data.redis.command) + self.assertIsNone(rs1_span.data.redis.error) + + self.assertIsNotNone(rs1_span.stack) + self.assertTrue(type(rs1_span.stack) is list) + self.assertGreater(len(rs1_span.stack), 0) + + # Redis span 2 + self.assertEqual('redis', rs2_span.n) + self.assertFalse('custom' in rs2_span.data.__dict__) + self.assertTrue('redis' in rs2_span.data.__dict__) + + self.assertEqual('redis-py', rs2_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs2_span.data.redis.connection) + self.assertEqual("SET", rs2_span.data.redis.command) + self.assertIsNone(rs2_span.data.redis.error) + + self.assertIsNotNone(rs2_span.stack) + self.assertTrue(type(rs2_span.stack) is list) + self.assertGreater(len(rs2_span.stack), 0) + + # Redis span 3 + self.assertEqual('redis', rs3_span.n) + self.assertFalse('custom' in rs3_span.data.__dict__) + self.assertTrue('redis' in rs3_span.data.__dict__) + + self.assertEqual('redis-py', rs3_span.data.redis.driver) + self.assertEqual("redis://%s/0" % testenv['redis_url'], rs3_span.data.redis.connection) + self.assertEqual("GET", rs3_span.data.redis.command) + self.assertIsNone(rs3_span.data.redis.error) + + self.assertIsNotNone(rs3_span.stack) + self.assertTrue(type(rs3_span.stack) is list) + self.assertGreater(len(rs3_span.stack), 0) + + def test_pipelined_requests(self): + result = None + with tracer.start_active_span('test'): + pipe = self.strict_redis.pipeline() + pipe.set('foox', 'barX') + pipe.set('fooy', 'barY') + pipe.get('foox') + result = pipe.execute() + + spans = self.recorder.queued_spans() + self.assertEqual(2, len(spans)) + + self.assertEqual([True, True, b'barX'], result) + + rs1_span = spans[0] + test_span = spans[1] + + self.assertIsNone(tracer.active_span) + + # Same traceId + self.assertEqual(test_span.t, rs1_span.t) + + # Parent relationships + self.assertEqual(rs1_span.p, test_span.s) + + # Error logging + self.assertFalse(test_span.error) + self.assertIsNone(test_span.ec) + self.assertFalse(rs1_span.error) + self.assertIsNone(rs1_span.ec) + + # Redis span 1 self.assertEqual('redis', rs1_span.n) self.assertFalse('custom' in rs1_span.data.__dict__) self.assertTrue('redis' in rs1_span.data.__dict__) self.assertEqual('redis-py', rs1_span.data.redis.driver) self.assertEqual("redis://%s/0" % testenv['redis_url'], rs1_span.data.redis.connection) + self.assertEqual("PIPELINE", rs1_span.data.redis.command) + self.assertEqual(['SET', 'SET', 'GET'], rs1_span.data.redis.subCommands) + self.assertIsNone(rs1_span.data.redis.error) self.assertIsNotNone(rs1_span.stack) self.assertTrue(type(rs1_span.stack) is list) From a647de6643131146ea7a26d29de8b5325f40ff08 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 13 Nov 2018 11:24:54 +0100 Subject: [PATCH 3/3] Fix error logging --- instana/instrumentation/redis.py | 4 ++-- instana/recorder.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/instana/instrumentation/redis.py b/instana/instrumentation/redis.py index 00082534..65c297c4 100644 --- a/instana/instrumentation/redis.py +++ b/instana/instrumentation/redis.py @@ -29,7 +29,7 @@ def execute_command_with_instana(wrapped, instance, args, kwargs): rv = wrapped(*args, **kwargs) except Exception as e: - scope.span.log_kv({'message': e}) + scope.span.set_tag("redis.error", str(e)) scope.span.set_tag("error", True) ec = scope.span.tags.get('ec', 0) scope.span.set_tag("ec", ec+1) @@ -66,7 +66,7 @@ def execute_with_instana(wrapped, instance, args, kwargs): rv = wrapped(*args, **kwargs) except Exception as e: - scope.span.log_kv({'message': e}) + scope.span.set_tag("redis.error", str(e)) scope.span.set_tag("error", True) ec = scope.span.tags.get('ec', 0) scope.span.set_tag("ec", ec+1) diff --git a/instana/recorder.py b/instana/recorder.py index 774e9e4d..bc8b56e5 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -120,7 +120,7 @@ def build_registered_span(self, span): data.redis = RedisData(connection=span.tags.pop('connection', None), driver=span.tags.pop('driver', None), command=span.tags.pop('command', None), - error=span.tags.pop('error', None), + error=span.tags.pop('redis.error', None), subCommands=span.tags.pop('subCommands', None)) if span.operation_name == "sqlalchemy":