diff --git a/pyvista/plotting/scalar_bars.py b/pyvista/plotting/scalar_bars.py
index 75d265273b..8d36201b32 100644
--- a/pyvista/plotting/scalar_bars.py
+++ b/pyvista/plotting/scalar_bars.py
@@ -422,6 +422,8 @@ def add_scalar_bar(self, title='', mapper=None, n_labels=5, italic=False,
 
         if label_font_size is not None or title_font_size is not None:
             scalar_bar.UnconstrainedFontSizeOn()
+            scalar_bar.AnnotationTextScalingOff()
+        else:
             scalar_bar.AnnotationTextScalingOn()
 
         label_text = scalar_bar.GetLabelTextProperty()
diff --git a/tests/plotting/image_cache/fixed_font_size_annotation_text_scaling_off.png b/tests/plotting/image_cache/fixed_font_size_annotation_text_scaling_off.png
new file mode 100644
index 0000000000..969b5dff35
Binary files /dev/null and b/tests/plotting/image_cache/fixed_font_size_annotation_text_scaling_off.png differ
diff --git a/tests/plotting/test_plotting.py b/tests/plotting/test_plotting.py
index 4e7cd74bd7..fa6dc699b6 100644
--- a/tests/plotting/test_plotting.py
+++ b/tests/plotting/test_plotting.py
@@ -1595,6 +1595,18 @@ def test_user_annotations_scalar_bar_mesh(uniform):
     p.show(before_close_callback=verify_cache_image)
 
 
+def test_fixed_font_size_annotation_text_scaling_off():
+    p = pyvista.Plotter()
+    sargs = {
+        'title_font_size': 12,
+        'label_font_size': 10}
+    p.add_mesh(examples.load_uniform(), clim=[100, 500], cmap='viridis',
+               below_color='blue', above_color='red',
+               annotations={300.: 'yum'},
+               scalar_bar_args=sargs)
+    p.show(before_close_callback=verify_cache_image)
+
+
 def test_user_annotations_scalar_bar_volume(uniform):
     p = pyvista.Plotter()
     p.add_volume(uniform, scalars='Spatial Point Data', annotations={100.: 'yum'})