diff --git a/promptsource/templates.py b/promptsource/templates.py
index 52425f266..ac5a6c72b 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -1,3 +1,4 @@
+import logging
 import os
 import random
 import uuid
@@ -380,6 +381,11 @@ def read_from_file(self) -> Dict:
         """
 
         if not os.path.exists(self.yaml_path):
+            dataset_name = f"{self.dataset_name} {self.subset_name}" if self.subset_name else self.dataset_name
+            logging.warning(
+                f"Tried instantiating `DatasetTemplates` for {dataset_name}, but no prompts found. "
+                "Please ignore this warning if you are creating new prompts for this dataset."
+            )
             return {}
         yaml_dict = yaml.load(open(self.yaml_path, "r"), Loader=yaml.FullLoader)
         return yaml_dict[self.TEMPLATES_KEY]
diff --git a/requirements.txt b/requirements.txt
index 29044fdfc..94d3b160c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-black
+black<=21.12b0
 datasets>=1.7.0
 flake8
 isort==5.8.0