yolo nas pose F.conv2d(input, weight, bias, self.stride, RuntimeError: expected scalar type Byte but found Float

[wesboyt]

🐛 Describe the bug

Hello I'm attempting to compare yolo nas pose with my custom yolov8 pose.
Please give me advice on training my model using my custom dataset, which is 1 class with 12 keypoints. Right now I am recieving the error:
venv\lib\site-packages\torch\nn\modules\conv.py", line 456, in _conv_forward
return F.conv2d(input, weight, bias, self.stride,
RuntimeError: expected scalar type Byte but found Float

I reused your animal example which im very confident is broken and pulled out the relevant config parameters. I'm also using a cvat server generated coco keypoints json file and your coco_pose_estimation_dataset: COCOPoseEstimationDataset.
It seems like its starting a training loop and failing the first forward call.

`import json
from typing import Union

import super_gradients
import yaml
from super_gradients.common.object_names import Models
from super_gradients.training import Trainer
from super_gradients.training.datasets.pose_estimation_datasets import coco_pose_estimation_dataset
from super_gradients.training.metrics import PoseEstimationMetrics
from super_gradients.training.models.pose_estimation_models import YoloNASPosePostPredictionCallback
from super_gradients.training.utils import EarlyStop
from super_gradients.training.utils.callbacks import ExtremeBatchPoseEstimationVisualizationCallback, Phase
from torch.utils.data import DataLoader
from super_gradients.training.datasets.pose_estimation_datasets import YoloNASPoseCollateFN
from super_gradients.training.metrics import PoseEstimationMetrics

CHECKPOINT_DIR = './pose'
trainer = Trainer(experiment_name='my_first_yn_pose_run', ckpt_root_dir=CHECKPOINT_DIR)

post_prediction_callback = YoloNASPosePostPredictionCallback(
pose_confidence_threshold = 0.01,
nms_iou_threshold = 0.7,
pre_nms_max_predictions = 300,
post_nms_max_predictions = 30,
)

metrics = PoseEstimationMetrics(
num_joints = 12,
oks_sigmas = [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1],
max_objects_per_image = 30,
post_prediction_callback = post_prediction_callback,
)

dataset = coco_pose_estimation_dataset.COCOPoseEstimationDataset(
data_dir='./pose/',
images_dir='images/',
json_file='annotations/person_keypoints_default.json',
include_empty_samples=True,
transforms=[],
edge_links=[],
edge_colors=[],
keypoint_colors=[]
)

visualization_callback = ExtremeBatchPoseEstimationVisualizationCallback(
keypoint_colors = [],
edge_colors = [],
edge_links = [],
loss_to_monitor = "YoloNASPoseLoss/loss",
max = True,
freq = 1,
max_images = 16,
enable_on_train_loader = True,
enable_on_valid_loader = True,
post_prediction_callback = post_prediction_callback,
)

early_stop = EarlyStop(
phase = Phase.VALIDATION_EPOCH_END,
monitor = "AP",
mode = "max",
min_delta = 0.0001,
patience = 100,
verbose = True,
)

#yolo_nas = super_gradients.training.models.get("yolo_nas_pose_l", pretrained_weights="coco_pose")
yolo_nas_pose = super_gradients.training.models.get(Models.YOLO_NAS_POSE_N, num_classes=1)

train_dataloader_params = {
'shuffle': True,
'batch_size': 3,
'drop_last': True,
'pin_memory': False,
'collate_fn': YoloNASPoseCollateFN()
}

train_params = {
"warmup_mode": "LinearBatchLRWarmup",
"warmup_initial_lr": 1e-8,
"lr_warmup_epochs": 2,
"initial_lr": 5e-5,
"lr_mode": "cosine",
"cosine_final_lr_ratio": 5e-3,
"max_epochs": 10,
"zero_weight_decay_on_bias_and_bn": True,
"batch_accumulate": 1,
"average_best_models": True,
"save_ckpt_epoch_list": [5, 10, 15, 20],
"loss": "yolo_nas_pose_loss",
"criterion_params": {
"oks_sigmas": [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1],
"classification_loss_weight": 1,
"classification_loss_type": "focal",
"regression_iou_loss_type": "ciou",
"iou_loss_weight": 2.5,
"dfl_loss_weight": 0.01,
"pose_cls_loss_weight": 1.0,
"pose_reg_loss_weight": 34.0,
"pose_classification_loss_type": "focal",
"rescale_pose_loss_with_assigned_score": True,
"assigner_multiply_by_pose_oks": True,
},
"optimizer": "AdamW",
"optimizer_params": {
"weight_decay": 0.000001
},
"ema": True,
"ema_params": {
"decay": 0.997,
"decay_type": "threshold"
},
"mixed_precision": True,
"sync_bn": False,
"valid_metrics_list": [metrics],
"phase_callbacks": [visualization_callback, early_stop],
"pre_prediction_callback": None,
"metric_to_watch": "AP",
"greater_metric_to_watch_is_better": True,
"convert": "all"
}

train_dataloader = DataLoader(dataset, **train_dataloader_params)
valid_dataloader = DataLoader(dataset, **train_dataloader_params)
trainer.train(model=yolo_nas_pose,
training_params=train_params,
train_loader=train_dataloader,
valid_loader=valid_dataloader,
)

`

Versions

I

[BloodAxe]

Greetings.
Usually this error message indicates that you are sending a uint8 type input to the model and not floats. Which may indicate that you don't have any image normalization. And indeed based on your code you are not using any transformations in training/validation datasets. You will get poor results.

Please follow your tutorial notebook on fine-tuning the YoloNAS-Pose: https://github.com/Deci-AI/super-gradients/blob/master/notebooks/YoloNAS_Pose_Fine_Tuning_Animals_Pose_Dataset.ipynb

[wesboyt]

thank you, will update with results.

Normalize in this context means turning pixel coordinates into floats proportional to the image?

[wesboyt]

Thank you, I was missing the normalization transform from the example. appreciate the prompt response sir!