diff --git a/notdiamond/llms/config.py b/notdiamond/llms/config.py
index e12d1d06..b5eff775 100644
--- a/notdiamond/llms/config.py
+++ b/notdiamond/llms/config.py
@@ -141,6 +141,10 @@ def from_string(cls, llm_provider: str):
             LLMConfig: initialized object with correct provider and model
         """
         split_items = llm_provider.split("/")
+        if len(split_items) != 2:
+            raise ValueError(
+                f"Expected string of format 'provider/model' but got {llm_provider}"
+            )
         provider = split_items[0]
         model = split_items[1]
         return cls(provider=provider, model=model)