[TensorRT] ERROR: Network must have at least one output

[NazarovAV]

Hi! I'm tring to convert GRCNN to tensorrt, but get error:

 import torch
 from torch2trt import torch2trt
 from GRCNN import GRCNN
 model = GRCNN(23)
 x = torch.ones(1,1,32,100).float().cuda()
 model = GRCNN(23).eval().cuda()
 model_trt = torch2trt(model, [x])

Traceback (most recent call last):
File "", line 1, in
File "/usr/local/lib/python3.6/dist-packages/torch2trt-0.1.0-py3.6-linux-aarch64.egg/torch2trt/torch2trt.py", line 436, in torch2trt
outputs = module(*inputs)
File "/home/nvidia/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(*input, **kwargs)
File "/home/nvidia/zmq_service_for_detectlp/grcnn/GRCNN.py", line 95, in forward
conv = conv.permute(2, 0, 1) # [w, b, c]
File "/usr/local/lib/python3.6/dist-packages/torch2trt-0.1.0-py3.6-linux-aarch64.egg/torch2trt/torch2trt.py", line 218, in wrapper
converter"converter"
File "/usr/local/lib/python3.6/dist-packages/torch2trt-0.1.0-py3.6-linux-aarch64.egg/torch2trt/converters/permute.py", line 17, in convert_permute
assert(permutation[0] == 0) # cannot move batch dim
AssertionError

If I convert grcnn to onnx I get warning, but grcnn.onnx is created:

torch.onnx.export(model, x, "grcnn.onnx", input_names=['input'], output_names=['output'], export_params=True) /home/nvidia/zmq_service_for_detectlp/grcnn/GRCNN.py:93: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
assert h == 1, "the height of conv must be 1"
/home/nvidia/.local/lib/python3.6/site-packages/torch/onnx/symbolic_opset9.py:1436: UserWarning: Exporting a model to ONNX with a batch_size other than 1, with a variable lenght with LSTM can cause an error when running the ONNX model with a different batch size. Make sure to save the model with a batch size of 1, or define the initial states (h0/c0) as inputs of the model.
"or define the initial states (h0/c0) as inputs of the model. ")

then, when I try convert onnx to tensorrt I get an error:

with open('grcnn.onnx', 'rb') as f:
    parser.parse(f.read())

False

engine = builder.build_cuda_engine(network)

[TensorRT] ERROR: Network must have at least one output

[zhaosj98]

You can try：
last_layer= network.get_layer(network.num_layers - 1)
if not last_layer.get_output(0):
network.mark_output(network.get_layer(network.num_layers - 1).get_output(0))
before engine = builder.build_cuda_engine(network)

[NazarovAV]

Hi @zhaosj98! Thanks for the answer!
I tried that

def build_engine(model_path):
    with trt.Builder(TRT_LOGGER) as builder, \
        builder.create_network() as network, \
        trt.OnnxParser(network, TRT_LOGGER) as parser: 
        builder.max_workspace_size = 1<<20
        builder.max_batch_size = 1
        with open(model_path, "rb") as f:
            parser.parse(f.read())
        last_layer= network.get_layer(network.num_layers - 1)
        if not last_layer.get_output(0):
            network.mark_output(network.get_layer(network.num_layers - 1).get_output(0))
        engine = builder.build_cuda_engine(network)
        return engine

but I got the error again

python3: ../builder/Network.cpp:863: virtual nvinfer1::ILayer* nvinfer1::Network::getLayer(int) const: Assertion `layerIndex >= 0' failed.

Aborted (core dumped)

[zhaosj98]

@nazzzaroff
You can refer to the following code to debug and modify yours.

import sys
import argparse
import logging
import os
import pickle
import time
import numpy as np
import tensorrt as trt
import torch
from torch2onnx import pretrained, maxinputlen, idim, odim

batchsize = 2
fp16 = True

def ONNX_build_engine(trt_model_name,onnx_model_name, batch_size=1):
  logger = logging.getLogger(__name__)
  TRT_LOGGER = trt.Logger()
  explicit_batch = 1 << (int)(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)#trt7
  with trt.Builder(TRT_LOGGER) as builder, builder.create_network(explicit_batch) as network, trt.OnnxParser(network, TRT_LOGGER) as parser:
    builder.max_batch_size = batchsize
    builder.max_workspace_size = 2 << 30
    print('Loading ONNX file from path {}...'.format(onnx_model_name))
    with open(onnx_model_name, 'rb') as model:
        print('Beginning ONNX file parsing')
        if not parser.parse(model.read()):
            for error in range(parser.num_errors):
                print(parser.get_error(error))
    if parser.num_errors == 0:
        print('Completed parsing of ONNX file')
        print('Building an engine from file {}; this may take a while...'.format(onnx_model_name))
 
        ####
        #builder.int8_mode = True
        #builder.int8_calibrator = calib
        if fp16:
            builder.fp16_mode = True
        ####
        print("num layers:", network.num_layers)
        #last_layer = network.get_layer(network.num_layers - 1)
        #if not last_layer.get_output(0):
        #  network.mark_output(network.get_layer(network.num_layers - 1).get_output(0))
        network.get_input(0).shape = [batchsize, maxinputlen, idim]#trt7
        network.get_input(1).shape = [batchsize, 1, maxinputlen]#trt7
        print(network.get_input(0).name, network.get_input(1).name)
        print(network.get_output(0).name, network.get_output(1).name)
        engine = builder.build_cuda_engine(network)
        print("engine:",engine)
        if engine is not None:
            print("Completed creating Engine")
            with open(trt_model_name, "wb") as f:
                f.write(engine.serialize())
            return engine
    else:
        print('Number of errors: {}'.format(parser.num_errors))
        for i in range(parser.num_errors):
           error = parser.get_error(i) # if it gets mnore than one error this have to be changed
           desc = error.desc()
           line = error.line()
           code = error.code()
           print('Description of the error: {}'.format(desc))
           print('Line where the error occurred: {}'.format(line))
           print('Error code: {}'.format(code))
        print("Model was not parsed successfully")


def main():
    print("Batchsize is {}. Use {} mode.".format(batchsize, "FP16" if fp16 else "FP32"))
    onnx_file = 'model_{}_maxinputlen{}.onnx'.format("pretrained" if pretrained else "random", maxinputlen)
    trt_file = 'model_{}_maxinputlen{}_batchsize{}_{}.trt'.format("pretrained" if pretrained else "random", maxinputlen, batchsize, "FP16" if fp16 else "FP32")
    trt.init_libnvinfer_plugins(None,'')
    ONNX_build_engine(trt_file, onnx_file, 1)

if __name__ == '__main__':
	main()

[zhaosj98]

@nazzzaroff

python3: ../builder/Network.cpp:863: virtual nvinfer1::ILayer* nvinfer1::Network::getLayer(int) const: Assertion `layerIndex >= 0' failed.

The reason for this error is that the number of layers in your parsed network is zero. So the index network.num_layers - 1 is -1, which is less than zero.

[NazarovAV]

@zhaosj98 Thanks!
I've tried your code with torchvision.models.resnet18 and get the error

import logging
import tensorrt as trt
# from torch2onnx import maxinputlen, idim

batchsize = 1
fp16 = True

def ONNX_build_engine(trt_model_name,onnx_model_name, batch_size=1):
  logger = logging.getLogger(__name__)
  TRT_LOGGER = trt.Logger()
  explicit_batch = 1 << (int)(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)#trt7
  with trt.Builder(TRT_LOGGER) as builder, builder.create_network(explicit_batch) as network, trt.OnnxParser(network, TRT_LOGGER) as parser:
    builder.max_batch_size = batchsize
    builder.max_workspace_size = 2 << 30
    print('Loading ONNX file from path {}...'.format(onnx_model_name))
    with open(onnx_model_name, 'rb') as model:
        print('Beginning ONNX file parsing')
        if not parser.parse(model.read()):
            for error in range(parser.num_errors):
                print(parser.get_error(error))
    if parser.num_errors == 0:
        print('Completed parsing of ONNX file')
        print('Building an engine from file {}; this may take a while...'.format(onnx_model_name))
 
        ####
        #builder.int8_mode = True
        #builder.int8_calibrator = calib
        if fp16:
            builder.fp16_mode = True
        ####
        print("num layers:", network.num_layers)
        #last_layer = network.get_layer(network.num_layers - 1)
        #if not last_layer.get_output(0):
        #  network.mark_output(network.get_layer(network.num_layers - 1).get_output(0))
        # network.get_input(0).shape = [batchsize, maxinputlen, idim]#trt7
        # network.get_input(1).shape = [batchsize, 1, maxinputlen]#trt7
        print(network.get_input(0).name, network.get_input(1).name)
        print(network.get_output(0).name, network.get_output(1).name)
        engine = builder.build_cuda_engine(network)
        print("engine:",engine)
        if engine is not None:
            print("Completed creating Engine")
            with open(trt_model_name, "wb") as f:
                f.write(engine.serialize())
            return engine
    else:
        print('Number of errors: {}'.format(parser.num_errors))
        for i in range(parser.num_errors):
           error = parser.get_error(i) # if it gets mnore than one error this have to be changed
           desc = error.desc()
           line = error.line()
           code = error.code()
           print('Description of the error: {}'.format(desc))
           print('Line where the error occurred: {}'.format(line))
           print('Error code: {}'.format(code))
        print("Model was not parsed successfully")


def main():
    onnx_file = "res.onnx"
    trt_file = "res.trt"
    trt.init_libnvinfer_plugins(None,'')
    ONNX_build_engine(trt_file, onnx_file, 1)

if __name__ == '__main__':
	main()

Loading ONNX file from path res.onnx...
Beginning ONNX file parsing
In node 0 (importModel): INVALID_GRAPH: Assertion failed: tensors.count(input_name)
Number of errors: 1
Description of the error: Assertion failed: tensors.count(input_name)
Line where the error occurred: 296
Error code: INVALID_GRAPH
Model was not parsed successfully

Did I convert resnet to onnx in a wrong way?

import torch 
import numpy as np
from torchvision.models import resnet18

onnx_path = 'res.onnx'

model = resnet18(pretrained=True)
model.eval()
model.cuda()

inputs = torch.ones(1,3,224,224).float().cuda()

outputs = model(inputs)

torch.onnx.export(model, inputs, onnx_path, 
                  input_names=['input'], output_names=['output'], 
                  export_params=True, 
                  dynamic_axes={'input' : {0 : 'batch_size'},
                                'output' : {1 : 'batch_size'}})

[zhaosj98]

I converted the pytorch model to onnx model, and then converted it to tensorrt engine successfully. Following is my modifications.

import logging
import tensorrt as trt
# from torch2onnx import maxinputlen, idim

batchsize = 1
fp16 = False

def ONNX_build_engine(trt_model_name,onnx_model_name, batch_size=1):
  logger = logging.getLogger(__name__)
  TRT_LOGGER = trt.Logger()
  explicit_batch = 1 << (int)(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)#trt7
  with trt.Builder(TRT_LOGGER) as builder, builder.create_network(explicit_batch) as network, trt.OnnxParser(network, TRT_LOGGER) as parser:
    builder.max_batch_size = batchsize
    builder.max_workspace_size = 2 << 30
    print('Loading ONNX file from path {}...'.format(onnx_model_name))
    with open(onnx_model_name, 'rb') as model:
        print('Beginning ONNX file parsing')
        if not parser.parse(model.read()):
            for error in range(parser.num_errors):
                print(parser.get_error(error))
    if parser.num_errors == 0:
        print('Completed parsing of ONNX file')
        print('Building an engine from file {}; this may take a while...'.format(onnx_model_name))
 
        ####
        #builder.int8_mode = True
        #builder.int8_calibrator = calib
        if fp16:
            builder.fp16_mode = True
        ####
        print("num layers:", network.num_layers)
        #last_layer = network.get_layer(network.num_layers - 1)
        #if not last_layer.get_output(0):
        #  network.mark_output(network.get_layer(network.num_layers - 1).get_output(0))
        network.get_input(0).shape = [batchsize, 3, 224, 224]#trt7
        #network.get_input(1).shape = [batchsize, 1, maxinputlen]#trt7
        #print(network.get_input(0).name, network.get_input(1).name)
        #print(network.get_output(0).name, network.get_output(1).name)
        engine = builder.build_cuda_engine(network)
        print("engine:",engine)
        if engine is not None:
            print("Completed creating Engine")
            with open(trt_model_name, "wb") as f:
                f.write(engine.serialize())
            return engine
    else:
        print('Number of errors: {}'.format(parser.num_errors))
        for i in range(parser.num_errors):
           error = parser.get_error(i) # if it gets mnore than one error this have to be changed
           desc = error.desc()
           line = error.line()
           code = error.code()
           print('Description of the error: {}'.format(desc))
           print('Line where the error occurred: {}'.format(line))
           print('Error code: {}'.format(code))
        print("Model was not parsed successfully")


def main():
    onnx_file = "res.onnx"
    trt_file = "res.trt"
    trt.init_libnvinfer_plugins(None,'')
    ONNX_build_engine(trt_file, onnx_file, 1)

if __name__ == '__main__':
	main()

The code for converting the pytorch model to onnx model is same as yours.

import torch 
import numpy as np
from torchvision.models import resnet18

onnx_path = 'res.onnx'

model = resnet18(pretrained=True)
model.eval()
model.cuda()

inputs = torch.ones(1,3,224,224).float().cuda()

outputs = model(inputs)

torch.onnx.export(model, inputs, onnx_path, 
                  input_names=['input'], output_names=['output'], 
                  export_params=True, 
                  dynamic_axes={'input' : {0 : 'batch_size'},
                                'output' : {1 : 'batch_size'}})

My tensorrt version is 7.1.3.4, cuda version is 10.2, cudnn version is 8.0, pytorch version is 1.6.0.
The error you got may be caused by collision of versions. Please check it.

[mk-nvidia]

Thanks for the helpful comments @zhaosj98
@nazzzaroff did that solve the error you were seeing?

[mk-nvidia]

Closing due to no response from user.