diff --git a/packages/components/src/palette-edit/index.js b/packages/components/src/palette-edit/index.js
index 2cfe84a944a61..0c4cbf88f6181 100644
--- a/packages/components/src/palette-edit/index.js
+++ b/packages/components/src/palette-edit/index.js
@@ -40,6 +40,8 @@ import { NavigableMenu } from '../navigable-container';
 import { DEFAULT_GRADIENT } from '../custom-gradient-picker/constants';
 import CustomGradientPicker from '../custom-gradient-picker';
 
+const DEFAULT_COLOR = '#000';
+
 function NameInput( { value, onChange, label } ) {
 	return (
 		<NameInputControl
@@ -51,6 +53,14 @@ function NameInput( { value, onChange, label } ) {
 	);
 }
 
+function getNameForPosition( position ) {
+	return sprintf(
+		/* translators: %s: is a temporary id for a custom color */
+		__( 'Color %s ' ),
+		position + 1
+	);
+}
+
 function Option( {
 	canOnlyChangeValues,
 	element,
@@ -143,6 +153,14 @@ function Option( {
 	);
 }
 
+function isTemporaryElement( slugPrefix, { slug, color, gradient }, index ) {
+	return (
+		slug === slugPrefix + kebabCase( getNameForPosition( index ) ) &&
+		( ( !! color && color === DEFAULT_COLOR ) ||
+			( !! gradient && gradient === DEFAULT_GRADIENT ) )
+	);
+}
+
 function PaletteEditListView( {
 	elements,
 	onChange,
@@ -159,9 +177,14 @@ function PaletteEditListView( {
 	}, [ elements ] );
 	useEffect( () => {
 		return () => {
-			if ( elementsReference.current.some( ( { slug } ) => ! slug ) ) {
+			if (
+				elementsReference.current.some( ( element, index ) =>
+					isTemporaryElement( slugPrefix, element, index )
+				)
+			) {
 				const newElements = elementsReference.current.filter(
-					( { slug } ) => slug
+					( element, index ) =>
+						! isTemporaryElement( slugPrefix, element, index )
 				);
 				onChange( newElements.length ? newElements : undefined );
 			}
@@ -272,19 +295,19 @@ export default function PaletteEdit( {
 									: __( 'Add color' )
 							}
 							onClick={ () => {
-								const tempOptionName = sprintf(
-									/* translators: %s: is a temporary id for a custom color */
-									__( 'Color %s ' ),
-									elementsLength + 1
+								const tempOptionName = getNameForPosition(
+									elementsLength
 								);
 								onChange( [
 									...elements,
 									{
 										...( isGradient
 											? { gradient: DEFAULT_GRADIENT }
-											: { color: '#000' } ),
+											: { color: DEFAULT_COLOR } ),
 										name: tempOptionName,
-										slug: '',
+										slug:
+											slugPrefix +
+											kebabCase( tempOptionName ),
 									},
 								] );
 								setIsEditing( true );