Internal change

PiperOrigin-RevId: 360883651

lra_benchmarks/image/configs/cifar10/base_cifar10_config.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -10,7 +10,7 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.\
+# limitations under the License.
 
 """Base Configuration."""
 

lra_benchmarks/image/configs/cifar10/bigbird_base.py

@@ -0,0 +1,34 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "bigbird"
+  config.model.block_size = 64
+  config.model.num_layers = 1
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_head = 4
+  config.model.classifier_pool = "CLS"
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/linear_transformer_base.py

@@ -0,0 +1,33 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "linear_transformer"
+  config.model.num_layers = 1
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_heads = 8
+  config.model.classifier_pool = "CLS"
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/linformer_base.py

@@ -0,0 +1,33 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "linformer"
+  config.model.num_layers = 1
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_heads = 8
+  config.model.classifier_pool = "CLS"
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/local_base.py

@@ -0,0 +1,34 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "local"
+  config.model.block_size = 64
+  config.model.num_layers = 1
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_heads = 2
+  config.model.classifier_pool = "MEAN"
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/longformer_base.py

@@ -0,0 +1,33 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "longformer"
+  config.model.num_layers = 4
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_heads = 4
+  config.model.classifier_pool = "MEAN"
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/performer_base.py

@@ -0,0 +1,34 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the default hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "performer"
+  config.model.num_layers = 1
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_heads = 8
+  config.model.classifier_pool = "CLS"
+
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/reformer_base.py

@@ -0,0 +1,34 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "reformer"
+  config.model.num_layers = 4
+  config.model.emb_dim = 64
+  config.model.qkv_dim = 32
+  config.model.mlp_dim = 64
+  config.model.num_heads = 8
+  config.model.classifier_pool = "CLS"
+
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/sinkhorn_base.py

@@ -0,0 +1,31 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "sinkhorn"
+  config.model.block_size = 64
+  config.model.num_layers = 1
+  config.weight_decay = 0.7
+  config.model.dropout_rate = 0.1
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/sparse_transformer_base.py

@@ -0,0 +1,33 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "sparse_transformer"
+  config.model.num_layers = 1
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_heads = 8
+  config.model.classifier_pool = "CLS"
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/synthesizer_base.py

@@ -0,0 +1,35 @@
+# Copyright 2021 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Configuration and hyperparameter sweeps."""
+
+from lra_benchmarks.image.configs.cifar10 import base_cifar10_config
+
+
+def get_config():
+  """Get the hyperparameter configuration."""
+  config = base_cifar10_config.get_config()
+  config.model_type = "synthesizer"
+  config.model.synthesizer_mode = "random"
+  config.model.num_layers = 1
+  config.model.emb_dim = 128
+  config.model.qkv_dim = 64
+  config.model.mlp_dim = 128
+  config.model.num_heads = 8
+  config.model.classifier_pool = "CLS"
+
+  return config
+
+
+def get_hyper(hyper):
+  return hyper.product([])

lra_benchmarks/image/configs/cifar10/transformer_base.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -10,7 +10,7 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.\
+# limitations under the License.
 
 """Configuration and hyperparameter sweeps."""