diff --git a/pkg/envtest/crd.go b/pkg/envtest/crd.go
index f9c58ea26a..b93a18a530 100644
--- a/pkg/envtest/crd.go
+++ b/pkg/envtest/crd.go
@@ -371,11 +371,16 @@ func modifyConversionWebhooks(crds []*apiextensionsv1.CustomResourceDefinition,
 		if crds[i].Spec.PreserveUnknownFields {
 			continue
 		}
-		// Continue if the GroupKind isn't registered as being convertible.
+		// Continue if the GroupKind isn't registered as being convertible,
+		// and remove any existing conversion webhooks if they exist.
+		// This is to prevent the CRD from being rejected by the apiserver, usually
+		// manifests that are generated by controller-gen will have a conversion
+		// webhook set, but we don't want to enable it if the type isn't registered.
 		if _, ok := convertibles[schema.GroupKind{
 			Group: crds[i].Spec.Group,
 			Kind:  crds[i].Spec.Names.Kind,
 		}]; !ok {
+			crds[i].Spec.Conversion = nil
 			continue
 		}
 		if crds[i].Spec.Conversion == nil {