diff --git a/astropy/visualization/wcsaxes/tests/test_misc.py b/astropy/visualization/wcsaxes/tests/test_misc.py
index 88a8418d398..c734880b250 100644
--- a/astropy/visualization/wcsaxes/tests/test_misc.py
+++ b/astropy/visualization/wcsaxes/tests/test_misc.py
@@ -19,6 +19,7 @@
     RectangularFrame,
     RectangularFrame1D,
 )
+from astropy.visualization.wcsaxes.ticklabels import TickLabels
 from astropy.visualization.wcsaxes.transforms import CurvedTransform
 from astropy.visualization.wcsaxes.utils import get_coord_meta
 from astropy.wcs import WCS
@@ -499,6 +500,43 @@ def test_simplify_labels_usetex(ignore_matplotlibrc, tmp_path):
     fig.savefig(tmp_path / "plot.png")
 
 
+@pytest.mark.parametrize(
+    "usetex, unicode_minus, label_str",
+    [
+        (True, True, "$-{}$"),
+        (True, False, "$-{}$"),
+        (False, True, "\N{MINUS SIGN}{}"),
+        (False, False, "-{}"),
+    ],
+)
+def test_simplify_labels_minus_sign(
+    ignore_matplotlibrc, usetex, unicode_minus, label_str
+):
+    # Ensure minus signs aren't removed from the front of labels across a grid of configuration possibilities
+    if usetex and TEX_UNAVAILABLE:
+        pytest.skip("TeX is unavailable")
+
+    ticklabels = TickLabels(None)
+    expected_labels = []
+    for i in range(1, 6):
+        label = label_str.format(i)
+        ticklabels.add(
+            axis="axis",
+            world=0,
+            angle=0,
+            text=label,
+            axis_displacement=0,
+            data=(i, i),
+        )
+        expected_labels.append(label)
+
+    with mpl.rc_context(
+        rc={"text.usetex": usetex, "axes.unicode_minus": unicode_minus}
+    ):
+        ticklabels.simplify_labels()
+    assert ticklabels.text["axis"] == expected_labels
+
+
 @pytest.mark.parametrize("frame_class", [RectangularFrame, EllipticalFrame])
 def test_set_labels_with_coords(ignore_matplotlibrc, frame_class):
     """Test if ``axis.set_xlabel()`` calls the correct ``coords[i]_set_axislabel()`` in a
diff --git a/astropy/visualization/wcsaxes/ticklabels.py b/astropy/visualization/wcsaxes/ticklabels.py
index bb3b866b8e5..a75dc86753f 100644
--- a/astropy/visualization/wcsaxes/ticklabels.py
+++ b/astropy/visualization/wcsaxes/ticklabels.py
@@ -126,7 +126,11 @@ def simplify_labels(self):
         """
         self.sort()
         skippable_chars = "0123456789."
-        skippable_chars += "\N{MINUS SIGN}" if rcParams["axes.unicode_minus"] else "-"
+        if rcParams["axes.unicode_minus"] and not rcParams["text.usetex"]:
+            skippable_chars += "\N{MINUS SIGN}"
+        else:
+            skippable_chars += "-"
+
         for axis in self.world:
             t1 = self.text[axis][0]
             for i in range(1, len(self.world[axis])):
diff --git a/docs/changes/visualization/16406.bugfix.rst b/docs/changes/visualization/16406.bugfix.rst
new file mode 100644
index 00000000000..30bdfcbbed4
--- /dev/null
+++ b/docs/changes/visualization/16406.bugfix.rst
@@ -0,0 +1 @@
+Fix a bug where ``WCSAxes`` could be missing negative signs on axis labels when using matplotlib's ``usetex`` mode.