diff --git a/backends/cadence/aot/graph_builder.py b/backends/cadence/aot/graph_builder.py
index 27604eac321..88ed2ac7699 100644
--- a/backends/cadence/aot/graph_builder.py
+++ b/backends/cadence/aot/graph_builder.py
@@ -38,7 +38,10 @@ def __init__(self) -> None:
         self.tracer: ExportPass.ExportTracer = self.ExportTracer(
             self, torch.fx.graph.CodeGen()
         )
-        self.fake_tensor_mode = FakeTensorMode(allow_fallback_kernels=False)
+        self.fake_tensor_mode = FakeTensorMode(
+            allow_fallback_kernels=False,
+            allow_non_fake_inputs=True,
+        )
         self.tracer.fake_tensor_mode = self.fake_tensor_mode
 
         # This will be called to create nodes in tracer.