don't cache TypeDecorator by default

The :class:`.TypeDecorator` class will now emit a warning when used in SQL
compilation with caching unless the ``.cache_ok`` flag is set to ``True``
or ``False``. ``.cache_ok`` indicates that all the parameters passed to the
object are safe to be used as a cache key, ``False`` means they are not.

Fixes: #6436
Change-Id: Ib1bb7dc4b124e38521d615c2e2e691e4915594fb

doc/build/changelog/unreleased_14/6436.rst

@@ -0,0 +1,10 @@
+.. change::
+    :tags: bug, sql, regression
+    :tickets: 6436
+
+    The :class:`.TypeDecorator` class will now emit a warning when used in SQL
+    compilation with caching unless the ``.cache_ok`` flag is set to ``True``
+    or ``False``. A new class-level attribute :attr:`.TypeDecorator.cache_ok`
+    may be set which will be used as an indication that all the parameters
+    passed to the object are safe to be used as a cache key if set to ``True``,
+    ``False`` means they are not.

doc/build/core/custom_types.rst

@@ -148,6 +148,7 @@ denormalize::
 
     class TZDateTime(TypeDecorator):
         impl = DateTime
+        cache_ok = True
 
         def process_bind_param(self, value, dialect):
             if value is not None:
@@ -186,6 +187,7 @@ binary in CHAR(16) if desired::
 
         """
         impl = CHAR
+        cache_ok = True
 
         def load_dialect_impl(self, dialect):
             if dialect.name == 'postgresql':
@@ -233,6 +235,8 @@ to/from JSON.   Can be modified to use Python's builtin json encoder::
 
         impl = VARCHAR
 
+        cache_ok = True
+
         def process_bind_param(self, value, dialect):
             if value is not None:
                 value = json.dumps(value)
@@ -306,6 +310,8 @@ method::
 
         impl = VARCHAR
 
+        cache_ok = True
+
         def coerce_compared_value(self, op, value):
             if op in (operators.like_op, operators.not_like_op):
                 return String()
@@ -416,8 +422,11 @@ transparently::
     class PGPString(TypeDecorator):
         impl = BYTEA
 
+        cache_ok = True
+
         def __init__(self, passphrase):
             super(PGPString, self).__init__()
+
             self.passphrase = passphrase
 
         def bind_expression(self, bindvalue):

lib/sqlalchemy/dialects/mssql/information_schema.py

@@ -25,6 +25,7 @@
 
 class CoerceUnicode(TypeDecorator):
     impl = Unicode
+    cache_ok = True
 
     def process_bind_param(self, value, dialect):
         if util.py2k and isinstance(value, util.binary_type):
@@ -211,6 +212,7 @@ class IdentitySqlVariant(TypeDecorator):
       correct value as string.
     """
     impl = Unicode
+    cache_ok = True
 
     def column_expression(self, colexpr):
         return cast(colexpr, Numeric)

lib/sqlalchemy/dialects/sqlite/pysqlite.py

@@ -312,6 +312,7 @@ def regexp(a, b):
 
     class MixedBinary(TypeDecorator):
         impl = String
+        cache_ok = True
 
         def process_result_value(self, value, dialect):
             if isinstance(value, str):

lib/sqlalchemy/sql/sqltypes.py

@@ -1780,6 +1780,7 @@ class PickleType(TypeDecorator):
     """
 
     impl = LargeBinary
+    cache_ok = True
 
     def __init__(
         self, protocol=pickle.HIGHEST_PROTOCOL, pickler=None, comparator=None
@@ -2027,6 +2028,7 @@ class Interval(Emulated, _AbstractInterval, TypeDecorator):
 
     impl = DateTime
     epoch = dt.datetime.utcfromtimestamp(0)
+    cache_ok = True
 
     def __init__(self, native=True, second_precision=None, day_precision=None):
         """Construct an Interval object.

lib/sqlalchemy/sql/traversals.py

@@ -152,7 +152,11 @@ def _gen_cache_key(self, anon_map, bindparams):
                 # efficient switch construct
 
                 if meth is STATIC_CACHE_KEY:
-                    result += (attrname, obj._static_cache_key)
+                    sck = obj._static_cache_key
+                    if sck is NO_CACHE:
+                        anon_map[NO_CACHE] = True
+                        return None
+                    result += (attrname, sck)
                 elif meth is ANON_NAME:
                     elements = util.preloaded.sql_elements
                     if isinstance(obj, elements._anonymous_label):

lib/sqlalchemy/sql/type_api.py

@@ -10,6 +10,7 @@
 """
 
 
+from sqlalchemy.sql.traversals import NO_CACHE
 from . import operators
 from .base import SchemaEventTarget
 from .visitors import Traversible
@@ -872,6 +873,8 @@ class MyType(types.TypeDecorator):
 
           impl = types.Unicode
 
+          cache_ok = True
+
           def process_bind_param(self, value, dialect):
               return "PREFIX:" + value
 
@@ -887,6 +890,16 @@ def copy(self, **kw):
     given; in this case, the ``impl`` variable can reference
     ``TypeEngine`` as a placeholder.
 
+    The :attr:`.TypeDecorator.cache_ok` class-level flag indicates if this
+    custom :class:`.TypeDecorator` is safe to be used as part of a cache key.
+    This flag defaults to ``None`` which will initially generate a warning
+    when the SQL compiler attempts to generate a cache key for a statement
+    that uses this type.  If the :class:`.TypeDecorator` is not guaranteed
+    to produce the same bind/result behavior and SQL generation
+    every time, this flag should be set to ``False``; otherwise if the
+    class produces the same behavior each time, it may be set to ``True``.
+    See :attr:`.TypeDecorator.cache_ok` for further notes on how this works.
+
     Types that receive a Python type that isn't similar to the ultimate type
     used may want to define the :meth:`TypeDecorator.coerce_compared_value`
     method. This is used to give the expression system a hint when coercing
@@ -946,6 +959,8 @@ def coerce_compared_value(self, op, value):
             class MyJsonType(TypeDecorator):
                 impl = postgresql.JSON
 
+                cache_ok = True
+
                 def coerce_compared_value(self, op, value):
                     return self.impl.coerce_compared_value(op, value)
 
@@ -1002,6 +1017,47 @@ def __init__(self, *args, **kwargs):
 
     """
 
+    cache_ok = None
+    """Indicate if statements using this :class:`.TypeDecorator` are "safe to
+    cache".
+
+    The default value ``None`` will emit a warning and then not allow caching
+    of a statement which includes this type.   Set to ``False`` to disable
+    statements using this type from being cached at all without a warning.
+    When set to ``True``, the object's class and selected elements from its
+    state will be used as part of the cache key, e.g.::
+
+        class MyType(TypeDecorator):
+            impl = String
+
+            cache_ok = True
+
+            def __init__(self, choices):
+                self.choices = tuple(choices)
+                self.internal_only = True
+
+    The cache key for the above type would be equivalent to::
+
+        (<class '__main__.MyType'>, ('choices', ('a', 'b', 'c')))
+
+    The caching scheme will extract attributes from the type that correspond
+    to the names of parameters in the ``__init__()`` method.  Above, the
+    "choices" attribute becomes part of the cache key but "internal_only"
+    does not, because there is no parameter named "internal_only".
+
+    The requirements for cacheable elements is that they are hashable
+    and also that they indicate the same SQL rendered for expressions using
+    this type every time for a given cache value.
+
+    .. versionadded:: 1.4.14 - added the ``cache_ok`` flag to allow
+       some configurability of caching for :class:`.TypeDecorator` classes.
+
+    .. seealso::
+
+        :ref:`sql_caching`
+
+    """
+
     class Comparator(TypeEngine.Comparator):
         """A :class:`.TypeEngine.Comparator` that is specific to
         :class:`.TypeDecorator`.
@@ -1037,6 +1093,21 @@ def comparator_factory(self):
                 {},
             )
 
+    @property
+    def _static_cache_key(self):
+        if self.cache_ok is None:
+            util.warn(
+                "TypeDecorator %r will not produce a cache key because "
+                "the ``cache_ok`` flag is not set to True.  "
+                "Set this flag to True if this type object's "
+                "state is safe to use in a cache key, or False to "
+                "disable this warning." % self
+            )
+        elif self.cache_ok is True:
+            return super(TypeDecorator, self)._static_cache_key
+
+        return NO_CACHE
+
     def _gen_dialect_impl(self, dialect):
         """
         #todo
@@ -1465,6 +1536,8 @@ class Variant(TypeDecorator):
 
     """
 
+    cache_ok = True
+
     def __init__(self, base, mapping):
         """Construct a new :class:`.Variant`.
 

lib/sqlalchemy/testing/suite/test_types.py

@@ -288,6 +288,7 @@ class _DateFixture(_LiteralRoundTripFixture, fixtures.TestBase):
     def define_tables(cls, metadata):
         class Decorated(TypeDecorator):
             impl = cls.datatype
+            cache_ok = True
 
         Table(
             "date_table",
@@ -477,6 +478,7 @@ class CastTypeDecoratorTest(_LiteralRoundTripFixture, fixtures.TestBase):
     def string_as_int(self):
         class StringAsInt(TypeDecorator):
             impl = String(50)
+            cache_ok = True
 
             def get_dbapi_type(self, dbapi):
                 return dbapi.NUMBER

test/dialect/mssql/test_types.py

@@ -1196,6 +1196,7 @@ def test_text_text_literal_binds(self):
 
 class MyPickleType(types.TypeDecorator):
     impl = PickleType
+    cache_ok = True
 
     def process_bind_param(self, value, dialect):
         if value:

test/dialect/mysql/test_compiler.py

@@ -693,6 +693,7 @@ def test_cast(self, type_, expected):
     def test_cast_type_decorator(self):
         class MyInteger(sqltypes.TypeDecorator):
             impl = Integer
+            cache_ok = True
 
         type_ = MyInteger()
         t = sql.table("t", sql.column("col"))

test/dialect/mysql/test_types.py

@@ -653,6 +653,7 @@ def test_boolean_roundtrip_reflected(
 
     class MyTime(TypeDecorator):
         impl = TIMESTAMP
+        cache_ok = True
 
     @testing.combinations(
         (TIMESTAMP,),

test/dialect/oracle/test_compiler.py

@@ -568,6 +568,7 @@ def test_for_update_of_w_limit_offset_adaption_partial_col_unpresent(self):
     def test_limit_preserves_typing_information(self):
         class MyType(TypeDecorator):
             impl = Integer
+            cache_ok = True
 
         stmt = select(type_coerce(column("x"), MyType).label("foo")).limit(1)
         dialect = oracle.dialect()

test/dialect/oracle/test_types.py

@@ -1189,6 +1189,7 @@ def test_setinputsizes(
 
         class TestTypeDec(TypeDecorator):
             impl = NullType()
+            cache_ok = True
 
             def load_dialect_impl(self, dialect):
                 if dialect.name == "oracle":

test/dialect/postgresql/test_dialect.py

@@ -1226,6 +1226,8 @@ def test_serial_integer(self):
         class BITD(TypeDecorator):
             impl = Integer
 
+            cache_ok = True
+
             def load_dialect_impl(self, dialect):
                 if dialect.name == "postgresql":
                     return BigInteger()

test/dialect/postgresql/test_on_conflict.py

@@ -42,6 +42,8 @@ def define_tables(cls, metadata):
         class SpecialType(sqltypes.TypeDecorator):
             impl = String
 
+            cache_ok = True
+
             def process_bind_param(self, value, dialect):
                 return value + " processed"
 

test/dialect/postgresql/test_types.py

@@ -746,6 +746,7 @@ def test_schema_reflection(self, metadata, connection):
     def test_custom_subclass(self, metadata, connection):
         class MyEnum(TypeDecorator):
             impl = Enum("oneHI", "twoHI", "threeHI", name="myenum")
+            cache_ok = True
 
             def process_bind_param(self, value, dialect):
                 if value is not None:
@@ -1391,6 +1392,7 @@ class ArrayRoundTripTest(object):
     def define_tables(cls, metadata):
         class ProcValue(TypeDecorator):
             impl = cls.ARRAY(Integer, dimensions=2)
+            cache_ok = True
 
             def process_bind_param(self, value, dialect):
                 if value is None:
@@ -2127,6 +2129,7 @@ def test_array_overlap_exec(self, connection):
 class _ArrayOfEnum(TypeDecorator):
     # previous workaround for array of enum
     impl = postgresql.ARRAY
+    cache_ok = True
 
     def bind_expression(self, bindvalue):
         return sa.cast(bindvalue, self)

test/dialect/test_sqlite.py

@@ -1682,6 +1682,7 @@ def test_sqlite_no_autoincrement(self):
     def test_sqlite_autoincrement_int_affinity(self):
         class MyInteger(sqltypes.TypeDecorator):
             impl = Integer
+            cache_ok = True
 
         table = Table(
             "autoinctable",
@@ -2693,6 +2694,7 @@ def define_tables(cls, metadata):
 
         class SpecialType(sqltypes.TypeDecorator):
             impl = String
+            cache_ok = True
 
             def process_bind_param(self, value, dialect):
                 return value + " processed"

test/engine/test_execute.py

@@ -336,6 +336,7 @@ class NonStandardException(OperationalError):
     def test_exception_wrapping_non_dbapi_statement(self):
         class MyType(TypeDecorator):
             impl = Integer
+            cache_ok = True
 
             def process_bind_param(self, value, dialect):
                 raise SomeException("nope")
@@ -539,6 +540,7 @@ class MyException(Exception, tsa.exc.DontWrapMixin):
 
         class MyType(TypeDecorator):
             impl = Integer
+            cache_ok = True
 
             def process_bind_param(self, value, dialect):
                 raise MyException("nope")
@@ -2575,6 +2577,7 @@ def test_exception_event_ad_hoc_context(self):
 
         class MyType(TypeDecorator):
             impl = Integer
+            cache_ok = True
 
             def process_bind_param(self, value, dialect):
                 raise nope

test/ext/mypy/files/type_decorator.py

@@ -12,6 +12,7 @@
 
 class IntToStr(TypeDecorator[int]):
     impl = String
+    cache_ok = True
 
     def process_bind_param(
         self,