deprecate LayerNormFp32

requirements-training.txt

@@ -1,4 +1,4 @@
-torch>=1.9.0
+torch>=1.10.0
 torchvision
 webdataset>=0.2.5
 regex

requirements.txt

@@ -1,4 +1,4 @@
-torch>=1.9.0
+torch>=1.10.0
 torchvision
 regex
 ftfy

src/open_clip/coca_model.py

@@ -7,7 +7,6 @@
 from dataclasses import dataclass
 
 from .transformer import (
-    LayerNormFp32,
     LayerNorm,
     QuickGELU,
     MultimodalTransformer,
@@ -58,9 +57,7 @@ def _build_text_decoder_tower(
 ):
     multimodal_cfg = MultimodalCfg(**multimodal_cfg) if isinstance(multimodal_cfg, dict) else multimodal_cfg
     act_layer = QuickGELU if quick_gelu else nn.GELU
-    norm_layer = (
-        LayerNormFp32 if cast_dtype in (torch.float16, torch.bfloat16) else LayerNorm
-    )
+    norm_layer = LayerNorm
 
     decoder = MultimodalTransformer(
         context_length=multimodal_cfg.context_length,

src/open_clip/model.py

@@ -18,7 +18,7 @@
 from .hf_model import HFTextEncoder
 from .modified_resnet import ModifiedResNet
 from .timm_model import TimmModel
-from .transformer import LayerNormFp32, LayerNorm, QuickGELU, Attention, VisionTransformer, TextTransformer,\
+from .transformer import LayerNorm, QuickGELU, Attention, VisionTransformer, TextTransformer,\
     text_global_pool
 from .utils import to_2tuple
 
@@ -139,7 +139,7 @@ def _build_vision_tower(
         )
     else:
         vision_heads = vision_cfg.width // vision_cfg.head_width
-        norm_layer = LayerNormFp32 if cast_dtype in (torch.float16, torch.bfloat16) else LayerNorm
+        norm_layer = LayerNorm
         if vision_cfg.norm_kwargs:
             norm_layer = partial(norm_layer, **vision_cfg.norm_kwargs)
         if vision_cfg.act_kwargs is not None:
@@ -190,7 +190,7 @@ def _build_text_tower(
         )
     else:
         act_layer = QuickGELU if quick_gelu else nn.GELU
-        norm_layer = LayerNormFp32 if cast_dtype in (torch.float16, torch.bfloat16) else LayerNorm
+        norm_layer = LayerNorm
         if text_cfg.norm_kwargs:
             norm_layer = partial(norm_layer, **text_cfg.norm_kwargs)
         if text_cfg.act_kwargs is not None:

src/open_clip/transformer.py

@@ -13,7 +13,10 @@
 
 
 class LayerNormFp32(nn.LayerNorm):
-    """Subclass torch's LayerNorm to handle fp16 (by casting to float32 and back)."""
+    """Subclass torch's LayerNorm to handle fp16 (by casting to float32 and back).
+
+    Deprecated: pytorch 1.10+ always performs LayerNorm in fp32. Retained for checkpoint compatibility.
+    """
 
     def forward(self, x: torch.Tensor):
         orig_type = x.dtype