[CI] Update bench script (#156)

* .

* .

* .

* format

.github/workflows/benchmark.yaml

@@ -1,7 +1,6 @@
 name: Benchmark
 on:
   workflow_dispatch:
-  pull_request:
   schedule:
     - cron: "0 3 * * *"
 jobs:
@@ -56,7 +55,7 @@ jobs:
 
       - name: Run benchmark
         run: |
-          python scripts/ci/benchmark.py --report result.txt --space 0 --git-commit ${{ github.sha }}
+          python scripts/ci/benchmark.py --report result.txt --space 2 --git-commit ${{ github.sha }}
 
       - name: Post result
         uses: peter-evans/create-or-update-comment@v2

python/hidet/graph/frontend/torch/dynamo_config.py

@@ -19,7 +19,7 @@ def __init__(self):
         self._use_fp16: bool = False
         self._use_fp16_reduction: bool = False
         self._use_cuda_graph: bool = True
-        self._use_tensor_core: bool = True
+        self._use_tensor_core: bool = False
         self._print_input_graph: bool = False
         self._dump_graph_ir: Optional[str] = None
         self._correctness_report: bool = False

scripts/ci/benchmark.py

@@ -29,10 +29,10 @@ def info(args) -> str:
 def main():
     args = parser.parse_args()
     commands = [
-        # f'hidet bench --space {args.space} --dtype float32 --report resnet50_f32.txt --tensor-core resnet --models resnet50',
-        # f'hidet bench --space {args.space} --dtype float16 --report resnet50_f16.txt --tensor-core resnet --models resnet50',
+        f'hidet bench --space {args.space} --dtype float32 --report resnet50_f32.txt --tensor-core resnet --models resnet50',
+        f'hidet bench --space {args.space} --dtype float16 --report resnet50_f16.txt --tensor-core resnet --models resnet50',
         f'hidet bench --space {args.space} --dtype float32 --report bert-seq128-f32.txt --tensor-core nlp --seq-length 128 --models bert-base-uncased',
-        # f'hidet bench --space {args.space} --dtype float16 --report bert-seq128-f16.txt --tensor-core nlp --seq-length 128 --models bert-base-uncased',
+        f'hidet bench --space {args.space} --dtype float16 --report bert-seq128-f16.txt --tensor-core nlp --seq-length 128 --models bert-base-uncased',
     ]
     with open(args.report, 'w') as f:
         t1 = time.time()

tests/frontends/torch/test_torch_conv2d.py

@@ -11,18 +11,26 @@
 # limitations under the License.
 import pytest
 import torch
+import torch.backends.cudnn
 from hidet.testing.torch_utils import check_module
 
 
 @pytest.mark.parametrize('in_shape,w_shape,stride,padding', [[[1, 3, 224, 224], [64, 3, 7, 7], 2, 3]])
 @pytest.mark.parametrize('dtype', [torch.float32])
 def test_conv2d(in_shape, w_shape, stride, padding, dtype):
+    torch.backends.cudnn.allow_tf32 = False  # disable tf32 for accuracy
     check_module(
         model=torch.nn.Conv2d(
-            in_channels=in_shape[1], out_channels=w_shape[0], kernel_size=w_shape[2:], stride=stride, padding=padding
+            in_channels=in_shape[1],
+            out_channels=w_shape[0],
+            kernel_size=w_shape[2:],
+            stride=stride,
+            padding=padding,
+            bias=False,
         ),
         args=[torch.randn(in_shape, dtype=dtype)],
     )
+    torch.backends.cudnn.allow_tf32 = True
 
 
 if __name__ == '__main__':

tests/frontends/torch/test_torch_resnet50.py

@@ -10,15 +10,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import torch
+import torch.backends.cudnn
 import pytest
 from hidet.testing.torch_utils import check_module
 
 
 @pytest.mark.parametrize('shape', [[1, 3, 224, 224]])
 def test_resnet50(shape):
+    torch.backends.cudnn.allow_tf32 = False  # disable tf32 for accuracy
     model = torch.hub.load('pytorch/vision:v0.6.0', 'resnet50', pretrained=True)
     x = torch.randn(*shape)
     check_module(model, [x], atol=1e-2, rtol=1e-2)
+    torch.backends.cudnn.allow_tf32 = True
 
 
 if __name__ == '__main__':