diff --git a/src/pyhf/pdf.py b/src/pyhf/pdf.py
index 50a94bfdbc..20123d81bb 100644
--- a/src/pyhf/pdf.py
+++ b/src/pyhf/pdf.py
@@ -464,7 +464,9 @@ def set_poi(self, name):
             raise exceptions.InvalidModel(
                 f"The parameter of interest '{name:s}' cannot be fit as it is not declared in the model specification."
             )
-        if dict(self.modifiers)[name] == "shapefactor":
+        if dict(self.modifiers).get(name, "") == "shapefactor":
+            # in the case of custom modifiers, the desired POI is in self.parameters but
+            # not in self.modifiers
             raise exceptions.InvalidModel(
                 f"The parameter '{name:s}' is of type 'shapefactor' and thus cannot be used as parameter of interest."
             )