diff --git a/index.js b/index.js
index 1b59cc95e..997c4ddc0 100644
--- a/index.js
+++ b/index.js
@@ -426,8 +426,10 @@ module.exports = app => {
               state === 'changes_requested' && config.labels.review[reviewerGroup].changesRequested,
             ],
             remove: [
-              config.labels.review[reviewerGroup].needsReview,
-              config.labels.review[reviewerGroup].requested,
+              (!hasRequestedReviewsForGroup || state === 'changes_requested') &&
+                config.labels.review[reviewerGroup].needsReview,
+              (!hasRequestedReviewsForGroup || state === 'changes_requested') &&
+                config.labels.review[reviewerGroup].requested,
               state === 'approved' &&
                 !hasRequestedReviewsForGroup &&
                 config.labels.review[reviewerGroup].changesRequested,