diff --git a/uarray/_backend.py b/uarray/_backend.py
index 6cc5b564..2f478519 100644
--- a/uarray/_backend.py
+++ b/uarray/_backend.py
@@ -142,8 +142,7 @@ def inner(*args, **kwargs):
             result = options.backend.__ua_function__(inner, a, kw)
 
             if result is NotImplemented:
-                with set_backend(options.backend, only=True, coerce=options.coerce):
-                    result = try_default(a, kw, options, errors)
+                result = try_default(a, kw, options, errors)
 
             if result is not NotImplemented:
                 break