diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 63383ea1..4e6ff4b1 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -92,8 +92,9 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const sizeLabels = ['size:s', 'size:m', 'size:l', 'size:xl', 'size:xxl'];
+            const newLabel = ${{ env.label }};
             const currentLabels = ${{ steps.fetch_labels.outputs.result }};
-            const labelsToRemove = currentLabels.filter(label => sizeLabels.includes(label));
+            const labelsToRemove = currentLabels.filter(label => sizeLabels.includes(label) && label !== newLabel);
             for (const label of labelsToRemove) {
               await github.rest.issues.removeLabel({
                 owner: context.repo.owner,