diff --git a/libexec/rbenv-gemset-delete b/libexec/rbenv-gemset-delete
index 17a6b84..7e3089a 100755
--- a/libexec/rbenv-gemset-delete
+++ b/libexec/rbenv-gemset-delete
@@ -8,5 +8,12 @@ if [ -z "$RBENV_VERSION" ] || [ -z "$RBENV_GEMSET" ]; then
   exit 1
 fi
 
-rm -rf "$(rbenv prefix "$RBENV_VERSION")/gemsets/$RBENV_GEMSET"
+GEMSET_PATH="$(rbenv prefix "$RBENV_VERSION")/gemsets/$RBENV_GEMSET"
+
+if [ ! -d $GEMSET_PATH ]; then
+  echo "couldn't delete ${RBENV_GEMSET} from ${RBENV_VERSION}. Check if it exists"
+  exit 1
+fi
+
+rm -rf $GEMSET_PATH
 echo "deleted ${RBENV_GEMSET} from ${RBENV_VERSION}"