[Performance] OpenVINO EP Produces incorrect inference results

[henxing]

Describe the issue

When using onnxruntime-openvino==1.16.0 on python3.8, I'm seeing inference results that do not match what the model produces in PyTorch, for certain models. I'm running with the OpenVINOExecutionProvider and using the GPU device. This seems to be a regression, as when I run with version 1.14.0, I do not see the difference in scores.

This is possibly related to #19975, where I previously left a comment.

To reproduce

See below for a script that reproduces the issue. The results of the BrokenModel will disagree, but the results of the other two models will be very close, if not the same.

import numpy as np
import onnxruntime as rt
import torch
from torch import nn


class BrokenModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv_1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1)
        self.conv_2 = nn.Conv2d(64, 1, kernel_size=1, stride=1, padding=0)

    def forward(self, x):
        x = self.conv_1(x)
        output = self.conv_2(x)
        return output.mean(dim=(1, 2, 3))


class BatchMeanModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv_1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1)
        self.conv_2 = nn.Conv2d(64, 1, kernel_size=1, stride=1, padding=0)

    def forward(self, x):
        x = self.conv_1(x)
        output = self.conv_2(x)
        return output.mean(dim=(1, 2, 3)), output.mean()


class FewChannelModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv_1 = nn.Conv2d(3, 3, kernel_size=3, stride=1, padding=1)
        self.conv_2 = nn.Conv2d(3, 1, kernel_size=1, stride=1, padding=0)

    def forward(self, x):
        x = self.conv_1(x)
        output = self.conv_2(x)
        return output.mean(dim=(1, 2, 3))


def run_model_pytorch_onnxruntime(arch, path):
    model = arch()
    model.eval()
    print("=" * 80)
    print(model)

    data = torch.ones(2, 3, 224, 224)
    data[0] *= 0

    print("Torch:")
    for _ in range(2):
        result = model(data)
        print(result)
    print()

    torch.onnx.export(
        model,
        data,
        path,
        input_names=["input"],
        output_names=["output"],
        export_params=True,
        dynamic_axes={name: {0: "batch_size"} for name in ("input", "output")},
        verbose=False,
    )

    sess_options = rt.SessionOptions()
    sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_DISABLE_ALL

    print("Onnxruntime:")
    rt_sess = rt.InferenceSession(
        path, sess_options, providers=["OpenVINOExecutionProvider"], provider_options=[{"device_id": "GPU"}]
    )
    for _ in range(2):
        outputs = rt_sess.run(None, {"input": data.numpy()})
        print(outputs)
    print()


if __name__ == "__main__":
    run_model_pytorch_onnxruntime(BrokenModel, "broken_model.onnx")
    print()
    run_model_pytorch_onnxruntime(BatchMeanModel, "batch_mean_model.onnx")
    print()
    run_model_pytorch_onnxruntime(FewChannelModel, "few_channel_model.onnx")

You'll need to install torch, onnxruntime-openvino, and numpy to run this script.

Urgency

No response

Platform

Linux

OS Version

Ubuntu 20.04

ONNX Runtime Installation

Released Package

ONNX Runtime Version or Commit ID

1.16.0

ONNX Runtime API

Python

Architecture

X64

Execution Provider

OpenVINO

Execution Provider Library Version

bundled with onnxruntime-openvino

Model File

No response

Is this a quantized model?

Yes

[github-actions]

This issue has been automatically marked as stale due to inactivity and will be closed in 30 days if no further activity occurs. If further support is needed, please provide an update and/or more details.