diff --git a/src/encoder.c b/src/encoder.c
index d711e11d..eab82634 100644
--- a/src/encoder.c
+++ b/src/encoder.c
@@ -225,6 +225,7 @@ sixel_parse_x_colorspec(
     }
 
     status = SIXEL_OK;
+
 end:
     sixel_allocator_free(allocator, buf);
 
@@ -1155,10 +1156,7 @@ sixel_encoder_new(
                                          env_default_bgcolor,
                                          allocator);
         if (SIXEL_FAILED(status)) {
-            sixel_allocator_free(allocator, *ppencoder);
-            sixel_allocator_unref(allocator);
-            *ppencoder = NULL;
-            goto end;
+            goto error;
         }
     }
 
@@ -1171,11 +1169,16 @@ sixel_encoder_new(
         }
     }
 
-    sixel_allocator_ref(allocator);
-
     /* success */
     status = SIXEL_OK;
 
+    goto end;
+
+error:
+    sixel_allocator_free(allocator, *ppencoder);
+    sixel_allocator_unref(allocator);
+    *ppencoder = NULL;
+
 end:
     return status;
 }