diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py
index f3878913a4e0b..220560d9238a9 100644
--- a/tests/test_modeling_tf_common.py
+++ b/tests/test_modeling_tf_common.py
@@ -1868,7 +1868,18 @@ def _generate_and_check_results(model, inputs_dict):
             generated = model.generate(inputs, **generate_kwargs).numpy()
             generate_xla = tf.function(model.generate, jit_compile=True)
             generated_xla = generate_xla(inputs, **generate_kwargs).numpy()
-            self.assertListEqual(generated.tolist(), generated_xla.tolist())
+
+            # Due to numerical instability, let's fail the test only if there are more than 10% of input sequences give
+            # different outputs between XLA and non-XLA versions. If there are less than 10 examples, let's be strict
+            # and not allow any difference.
+            diff = [[], []]
+            for _generated, _generated_xla in zip(generated.tolist(), generated_xla.tolist()):
+                if _generated != _generated_xla:
+                    diff[0].append(_generated)
+                    diff[1].append(_generated_xla)
+            ratio = len(diff[0]) / len(generated)
+            if ratio > 0.1 or (len(diff[0]) > 0 and len(generated) < 10):
+                self.assertListEqual(diff[0], diff[1])
 
         for model_class in self.all_generative_model_classes:
             config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()