Add back time series transformer and x-clip

huggingface · May 24, 2024 · 9bc7863 · 9bc7863
1 parent 0101b2c
commit 9bc7863
Show file tree

Hide file tree

Showing 25 changed files with 1,432 additions and 150 deletions.
diff --git a/docs/source/en/model_doc/time_series_transformer.md b/docs/source/en/model_doc/time_series_transformer.md
@@ -16,14 +16,6 @@ rendered properly in your Markdown viewer.
 
 # Time Series Transformer
 
-<Tip warning={true}>
-
-This model is in maintenance mode only, we don't accept any new PRs changing its code.
-If you run into any issues running this model, please reinstall the last version that supported this model: v4.40.2.
-You can do so by running the following command: `pip install -U transformers==4.40.2`.
-
-</Tip>
-
 ## Overview
 
 The Time Series Transformer model is a vanilla encoder-decoder Transformer for time series forecasting.

diff --git a/docs/source/en/model_doc/xclip.md b/docs/source/en/model_doc/xclip.md
@@ -16,14 +16,6 @@ rendered properly in your Markdown viewer.
 
 # X-CLIP
 
-<Tip warning={true}>
-
-This model is in maintenance mode only, we don't accept any new PRs changing its code.
-If you run into any issues running this model, please reinstall the last version that supported this model: v4.40.2.
-You can do so by running the following command: `pip install -U transformers==4.40.2`.
-
-</Tip>
-
 ## Overview
 
 The X-CLIP model was proposed in [Expanding Language-Image Pretrained Models for General Video Recognition](https://arxiv.org/abs/2208.02816) by Bolin Ni, Houwen Peng, Minghao Chen, Songyang Zhang, Gaofeng Meng, Jianlong Fu, Shiming Xiang, Haibin Ling.

diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
@@ -360,7 +360,6 @@
         "Speech2Text2Tokenizer",
     ],
     "models.deprecated.tapex": ["TapexTokenizer"],
-    "models.deprecated.time_series_transformer": ["TimeSeriesTransformerConfig"],
     "models.deprecated.trajectory_transformer": ["TrajectoryTransformerConfig"],
     "models.deprecated.transfo_xl": [
         "TransfoXLConfig",
@@ -374,12 +373,6 @@
     ],
     "models.deprecated.van": ["VanConfig"],
     "models.deprecated.vit_hybrid": ["ViTHybridConfig"],
-    "models.deprecated.x_clip": [
-        "XCLIPConfig",
-        "XCLIPProcessor",
-        "XCLIPTextConfig",
-        "XCLIPVisionConfig",
-    ],
     "models.deprecated.xlm_prophetnet": ["XLMProphetNetConfig"],
     "models.depth_anything": ["DepthAnythingConfig"],
     "models.detr": ["DetrConfig"],
@@ -719,6 +712,7 @@
         "TapasConfig",
         "TapasTokenizer",
     ],
+    "models.time_series_transformer": ["TimeSeriesTransformerConfig"],
     "models.timesformer": ["TimesformerConfig"],
     "models.timm_backbone": ["TimmBackboneConfig"],
     "models.trocr": [
@@ -787,6 +781,12 @@
         "WhisperProcessor",
         "WhisperTokenizer",
     ],
+    "models.x_clip": [
+        "XCLIPConfig",
+        "XCLIPProcessor",
+        "XCLIPTextConfig",
+        "XCLIPVisionConfig",
+    ],
     "models.xglm": ["XGLMConfig"],
     "models.xlm": ["XLMConfig", "XLMTokenizer"],
     "models.xlm_roberta": ["XLMRobertaConfig"],
@@ -1890,13 +1890,6 @@
     _import_structure["models.deprecated.speech_to_text_2"].extend(
         ["Speech2Text2ForCausalLM", "Speech2Text2PreTrainedModel"]
     )
-    _import_structure["models.deprecated.time_series_transformer"].extend(
-        [
-            "TimeSeriesTransformerForPrediction",
-            "TimeSeriesTransformerModel",
-            "TimeSeriesTransformerPreTrainedModel",
-        ]
-    )
     _import_structure["models.deprecated.trajectory_transformer"].extend(
         [
             "TrajectoryTransformerModel",
@@ -1935,14 +1928,6 @@
             "ViTHybridPreTrainedModel",
         ]
     )
-    _import_structure["models.deprecated.x_clip"].extend(
-        [
-            "XCLIPModel",
-            "XCLIPPreTrainedModel",
-            "XCLIPTextModel",
-            "XCLIPVisionModel",
-        ]
-    )
     _import_structure["models.deprecated.xlm_prophetnet"].extend(
         [
             "XLMProphetNetDecoder",
@@ -3197,6 +3182,13 @@
             "load_tf_weights_in_tapas",
         ]
     )
+    _import_structure["models.time_series_transformer"].extend(
+        [
+            "TimeSeriesTransformerForPrediction",
+            "TimeSeriesTransformerModel",
+            "TimeSeriesTransformerPreTrainedModel",
+        ]
+    )
     _import_structure["models.timesformer"].extend(
         [
             "TimesformerForVideoClassification",
@@ -3416,6 +3408,14 @@
             "WhisperPreTrainedModel",
         ]
     )
+    _import_structure["models.x_clip"].extend(
+        [
+            "XCLIPModel",
+            "XCLIPPreTrainedModel",
+            "XCLIPTextModel",
+            "XCLIPVisionModel",
+        ]
+    )
     _import_structure["models.xglm"].extend(
         [
             "XGLMForCausalLM",
@@ -4906,9 +4906,6 @@
         Speech2Text2Tokenizer,
     )
     from .models.deprecated.tapex import TapexTokenizer
-    from .models.deprecated.time_series_transformer import (
-        TimeSeriesTransformerConfig,
-    )
     from .models.deprecated.trajectory_transformer import (
         TrajectoryTransformerConfig,
     )
@@ -4926,12 +4923,6 @@
     from .models.deprecated.vit_hybrid import (
         ViTHybridConfig,
     )
-    from .models.deprecated.x_clip import (
-        XCLIPConfig,
-        XCLIPProcessor,
-        XCLIPTextConfig,
-        XCLIPVisionConfig,
-    )
     from .models.deprecated.xlm_prophetnet import (
         XLMProphetNetConfig,
     )
@@ -5307,6 +5298,9 @@
         TapasConfig,
         TapasTokenizer,
     )
+    from .models.time_series_transformer import (
+        TimeSeriesTransformerConfig,
+    )
     from .models.timesformer import (
         TimesformerConfig,
     )
@@ -5384,6 +5378,12 @@
         WhisperProcessor,
         WhisperTokenizer,
     )
+    from .models.x_clip import (
+        XCLIPConfig,
+        XCLIPProcessor,
+        XCLIPTextConfig,
+        XCLIPVisionConfig,
+    )
     from .models.xglm import XGLMConfig
     from .models.xlm import XLMConfig, XLMTokenizer
     from .models.xlm_roberta import (
@@ -6375,11 +6375,6 @@
             Speech2Text2ForCausalLM,
             Speech2Text2PreTrainedModel,
         )
-        from .models.deprecated.time_series_transformer import (
-            TimeSeriesTransformerForPrediction,
-            TimeSeriesTransformerModel,
-            TimeSeriesTransformerPreTrainedModel,
-        )
         from .models.deprecated.trajectory_transformer import (
             TrajectoryTransformerModel,
             TrajectoryTransformerPreTrainedModel,
@@ -6408,12 +6403,6 @@
             ViTHybridModel,
             ViTHybridPreTrainedModel,
         )
-        from .models.deprecated.x_clip import (
-            XCLIPModel,
-            XCLIPPreTrainedModel,
-            XCLIPTextModel,
-            XCLIPVisionModel,
-        )
         from .models.deprecated.xlm_prophetnet import (
             XLMProphetNetDecoder,
             XLMProphetNetEncoder,
@@ -7406,6 +7395,11 @@
             TapasPreTrainedModel,
             load_tf_weights_in_tapas,
         )
+        from .models.time_series_transformer import (
+            TimeSeriesTransformerForPrediction,
+            TimeSeriesTransformerModel,
+            TimeSeriesTransformerPreTrainedModel,
+        )
         from .models.timesformer import (
             TimesformerForVideoClassification,
             TimesformerModel,
@@ -7571,6 +7565,12 @@
             WhisperModel,
             WhisperPreTrainedModel,
         )
+        from .models.x_clip import (
+            XCLIPModel,
+            XCLIPPreTrainedModel,
+            XCLIPTextModel,
+            XCLIPVisionModel,
+        )
         from .models.xglm import (
             XGLMForCausalLM,
             XGLMModel,

diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py
@@ -218,6 +218,7 @@
     t5,
     table_transformer,
     tapas,
+    time_series_transformer,
     timesformer,
     timm_backbone,
     trocr,
@@ -249,6 +250,7 @@
     wav2vec2_with_lm,
     wavlm,
     whisper,
+    x_clip,
     xglm,
     xlm,
     xlm_roberta,

diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py
@@ -601,13 +601,11 @@
     "retribert",
     "speech_to_text_2",
     "tapex",
-    "time_series_transformer",
     "trajectory_transformer",
     "transfo_xl",
     "tvlt",
     "van",
     "vit_hybrid",
-    "x_clip",
     "xlm_prophetnet",
 ]
 

diff --git a/src/transformers/models/autoformer/modeling_autoformer.py b/src/transformers/models/autoformer/modeling_autoformer.py
@@ -14,7 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" PyTorch Autoformer model."""
+"""PyTorch Autoformer model."""
 
 import math
 from dataclasses import dataclass
@@ -167,7 +167,7 @@ class AutoformerModelOutput(ModelOutput):
     static_features: Optional[torch.FloatTensor] = None
 
 
-# Copied from transformers.models.informer.modeling_informer.InformerFeatureEmbedder with Informer->Autoformer
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesFeatureEmbedder with TimeSeries->Autoformer
 class AutoformerFeatureEmbedder(nn.Module):
     """
     Embed a sequence of categorical features.
@@ -202,7 +202,7 @@ def forward(self, features: torch.Tensor) -> torch.Tensor:
         )
 
 
-# Copied from transformers.models.informer.modeling_informer.InformerStdScaler with InformerTransformer->Autoformer,Informer->Autoformer
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesStdScaler with TimeSeriesTransformer->Autoformer,TimeSeries->Autoformer
 class AutoformerStdScaler(nn.Module):
     """
     Standardize features by calculating the mean and scaling along the first dimension, and then normalizes it by
@@ -238,7 +238,7 @@ def forward(
         return (data - loc) / scale, loc, scale
 
 
-# Copied from transformers.models.informer.modeling_informer.InformerMeanScaler with InformerTransformer->Autoformer,Informer->Autoformer
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesMeanScaler with TimeSeriesTransformer->Autoformer,TimeSeries->Autoformer
 class AutoformerMeanScaler(nn.Module):
     """
     Computes a scaling factor as the weighted average absolute value along the first dimension, and scales the data
@@ -293,7 +293,7 @@ def forward(
         return scaled_data, torch.zeros_like(scale), scale
 
 
-# Copied from transformers.models.informer.modeling_informer.InformerNOPScaler with InformerTransformer->Autoformer,Informer->Autoformer
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesNOPScaler with TimeSeriesTransformer->Autoformer,TimeSeries->Autoformer
 class AutoformerNOPScaler(nn.Module):
     """
     Assigns a scaling factor equal to 1 along the first dimension, and therefore applies no scaling to the input data.
@@ -321,7 +321,7 @@ def forward(
         return data, loc, scale
 
 
-# Copied from transformers.models.informer.modeling_informer.weighted_average
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.weighted_average
 def weighted_average(input_tensor: torch.Tensor, weights: Optional[torch.Tensor] = None, dim=None) -> torch.Tensor:
     """
     Computes the weighted average of a given tensor across a given `dim`, masking values associated with weight zero,
@@ -346,7 +346,7 @@ def weighted_average(input_tensor: torch.Tensor, weights: Optional[torch.Tensor]
         return input_tensor.mean(dim=dim)
 
 
-# Copied from transformers.models.informer.modeling_informer.nll
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.nll
 def nll(input: torch.distributions.Distribution, target: torch.Tensor) -> torch.Tensor:
     """
     Computes the negative log likelihood loss from input distribution with respect to target.
@@ -389,7 +389,7 @@ def forward(self, input_ids_shape: torch.Size, past_key_values_length: int = 0)
         return super().forward(positions)
 
 
-# Copied from transformers.models.informer.modeling_informer.InformerValueEmbedding with Informer->Autoformer
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesValueEmbedding with TimeSeries->Autoformer
 class AutoformerValueEmbedding(nn.Module):
     def __init__(self, feature_size, d_model):
         super().__init__()
@@ -1059,6 +1059,7 @@ def _init_weights(self, module):
 """
 
 
+# Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesTransformerEncoder with TimeSeriesTransformer->Autoformer,TimeSeries->Autoformer
 class AutoformerEncoder(AutoformerPreTrainedModel):
     """
     Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a

diff --git a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py
@@ -12,8 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" PyTorch BlenderbotSmall model."""
-
+"""PyTorch BlenderbotSmall model."""
 
 import copy
 import math
@@ -314,7 +313,7 @@ def forward(
         return outputs
 
 
-# TODO: Implement attention with SDPA for TTransformer.
+# TODO: Implement attention with SDPA for TimeSeriesTransformer.
 BLENDERBOT_SMALL_ATTENTION_CLASSES = {
     "eager": BlenderbotSmallAttention,
 }