Revert "inductor: only do the conv+bn folding for the freezing path (#…

…109270)"

This reverts commit c7017ff.

Reverted #109270 on behalf of https://github.com/malfet due to Broke slow test, see https://hud.pytorch.org/pytorch/pytorch/commit/c7017fff38e73210541124739ed9404492ddd68c ([comment](#109270 (comment)))

torch/_inductor/fx_passes/pre_grad.py

@@ -89,11 +89,12 @@ def fuse_fx(gm: torch.fx.GraphModule, example_inputs):
         gm = permute_matmul_fusion(gm)
 
     # make sure the autograd is disabled.
-    if torch.is_grad_enabled() or not is_cpu:
+    if torch.is_grad_enabled():
         return gm
-    if config.freezing:
-        gm = remove_identity(gm)
-        gm = fuse_conv_bn(gm)
+    if not is_cpu:
+        return gm
+    gm = remove_identity(gm)
+    gm = fuse_conv_bn(gm)
     return gm