RuntimeError in DataLoader: Stack Expects Each Tensor to Be Equal Size During Training

[Chap5732]

Is there an existing issue for this?

• I have searched the existing issues

Bug description

My training dataset consists of 338 images with 1-10 mice, 6 bodyparts were annotated.
(nose, left_ear, right_ear, mid_back, mouse_center and tail1, just the name I set, they are not truly the case as in superanimal model according to the result showed in confusion_matrix.png)

And I wanted to use a superanimal pretrained model (sa-tvm, 27 kpts) to fine-tune on my data(6 kpts) and I got error as follows:
Traceback (most recent call last): File "/ssd01/user_acc_data/oppa/deeplabcut/code/satvm_training.py", line 29, in <module> main() File "/ssd01/user_acc_data/oppa/deeplabcut/code/satvm_training.py", line 26, in main train_network(config_path, shuffle, device, net_type) File "/ssd01/user_acc_data/oppa/deeplabcut/code/satvm_training.py", line 6, in train_network deeplabcut.train_network( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/compat.py", line 245, in train_network return train_network( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/train.py", line 326, in train_network train( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/train.py", line 189, in train runner.fit( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/runners/train.py", line 170, in fit train_loss = self._epoch( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/runners/train.py", line 220, in _epoch for i, batch in enumerate(loader): File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 631, in __next__ data = self._next_data() File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1346, in _next_data return self._process_data(data) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1372, in _process_data data.reraise() File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/_utils.py", line 705, in reraise raise exception RuntimeError: Caught RuntimeError in DataLoader worker process 0. Original Traceback (most recent call last): File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 308, in _worker_loop data = fetcher.fetch(index) # type: ignore[possibly-undefined] File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch return self.collate_fn(data) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 316, in default_collate return collate(batch, collate_fn_map=default_collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 154, in collate clone.update({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 154, in <dictcomp> clone.update({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 141, in collate return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 222, in collate_numpy_array_fn return collate([torch.as_tensor(b) for b in batch], collate_fn_map=collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 141, in collate return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 213, in collate_tensor_fn return torch.stack(batch, 0, out=out) RuntimeError: stack expects each tensor to be equal size, but got [3, 1688, 1695] at entry 0 and [3, 1688, 1726] at entry 11

Operating System

operating system

Linux

DeepLabCut version

dlc version

3.0.0.rc2

DeepLabCut mode

multi animal

Device type

gpu

Steps To Reproduce

create shuffle step：

from pathlib import Path
import deeplabcut
from deeplabcut.core.engine import Engine
from deeplabcut.core.weight_init import WeightInitialization
from deeplabcut.modelzoo.utils import (
    create_conversion_table,
    read_conversion_table_from_csv,
)
from deeplabcut.utils.pseudo_label import keypoint_matching

def create_training_dataset(config_path, shuffle, super_animal_name, model_name, conversion_table_path, net_type):
    # Step 1: Keypoint matching before creating the training dataset
    keypoint_matching(config_path, super_animal_name, model_name)

    # Step 2: Initialize weights for memory replay
    table = create_conversion_table(
        config=config_path,
        super_animal=super_animal_name,
        project_to_super_animal=read_conversion_table_from_csv(conversion_table_path),
    )
    
    weight_init = WeightInitialization(
        dataset=super_animal_name,
        conversion_array=table.to_array(),
        with_decoder=True,
        memory_replay=True,
    )

    # Step 3: Create training dataset
    deeplabcut.create_training_dataset(
        config_path,
        Shuffles=[shuffle],
        net_type=net_type,
        weight_init=weight_init,
        engine=Engine.PYTORCH,
        userfeedback=False
    )

def main():
    dlc_proj_root = Path("/ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23")
    super_animal_name = "superanimal_topviewmouse"
    net_type = 'top_down_hrnet_w32'
    model_name = 'hrnetw32'
    shuffle = 1

    config_path = str(dlc_proj_root / "config.yaml")
    conversion_table_path = dlc_proj_root / "memory_replay" / "conversion_table.csv"

    # Step 1: Create training dataset
    create_training_dataset(config_path, shuffle, super_animal_name, model_name, conversion_table_path, net_type)

if __name__ == "__main__":
    main()

training step：
`from pathlib import Path
import deeplabcut

def train_network(config_path, shuffle, device, net_type):
    # Train the network with memory replay
    deeplabcut.train_network(
        config_path, 
        shuffle=shuffle, 
        device=device, 
        pose_threshold=0.1, 
        net_type=net_type,
        detector_batch_size=32,
        batch_size=64,
        freeze_bn_stats=False
    )

def main():
    dlc_proj_root = Path("/ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23")
    net_type = 'top_down_hrnet_w32'
    device = "cuda"
    shuffle = 1

    config_path = str(dlc_proj_root / "config.yaml")

    # Step 2: Train the network
    train_network(config_path, shuffle, device, net_type)

if __name__ == "__main__":
    main()

error report：
-------------------------------------------------- Traceback (most recent call last): File "/ssd01/user_acc_data/oppa/deeplabcut/code/satvm_training.py", line 29, in <module> main() File "/ssd01/user_acc_data/oppa/deeplabcut/code/satvm_training.py", line 26, in main train_network(config_path, shuffle, device, net_type) File "/ssd01/user_acc_data/oppa/deeplabcut/code/satvm_training.py", line 6, in train_network deeplabcut.train_network( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/compat.py", line 245, in train_network return train_network( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/train.py", line 326, in train_network train( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/train.py", line 189, in train runner.fit( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/runners/train.py", line 170, in fit train_loss = self._epoch( File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/runners/train.py", line 220, in _epoch for i, batch in enumerate(loader): File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 631, in __next__ data = self._next_data() File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1346, in _next_data return self._process_data(data) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1372, in _process_data data.reraise() File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/_utils.py", line 705, in reraise raise exception RuntimeError: Caught RuntimeError in DataLoader worker process 0. Original Traceback (most recent call last): File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 308, in _worker_loop data = fetcher.fetch(index) # type: ignore[possibly-undefined] File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch return self.collate_fn(data) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 316, in default_collate return collate(batch, collate_fn_map=default_collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 154, in collate clone.update({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 154, in <dictcomp> clone.update({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 141, in collate return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 222, in collate_numpy_array_fn return collate([torch.as_tensor(b) for b in batch], collate_fn_map=collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 141, in collate return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 213, in collate_tensor_fn return torch.stack(batch, 0, out=out) RuntimeError: stack expects each tensor to be equal size, but got [3, 1688, 1695] at entry 0 and [3, 1688, 1726] at entry 11

Relevant log output

pytorch_config:

Task:
scorer:
date:
multianimalproject:
identity:
project_path:
engine: tensorflow
video_sets:
bodyparts:
start:
stop:
numframes2pick:
skeleton: []
skeleton_color: black
pcutoff:
dotsize:
alphavalue:
colormap:
TrainingFraction:
iteration:
default_net_type:
default_augmenter:
snapshotindex:
detector_snapshotindex:
batch_size:
cropping:
x1:
x2:
y1:
y2:
corner2move2:
move2corner:
SuperAnimalConversionTables:
data:
  colormode: RGB
  inference:
    auto_padding:
      pad_width_divisor: 32
      pad_height_divisor: 32
    normalize_images: true
  train:
    affine:
      p: 0.5
      scaling:
      - 1.0
      - 1.0
      rotation: 30
      translation: 0
    gaussian_noise: 12.75
    normalize_images: true
    auto_padding:
      pad_width_divisor: 32
      pad_height_divisor: 32
detector:
  data:
    colormode: RGB
    inference:
      normalize_images: true
    train:
      hflip: true
      normalize_images: true
  device: auto
  model:
    type: FasterRCNN
    variant: fasterrcnn_resnet50_fpn_v2
    box_score_thresh: 0.6
    pretrained: false
  runner:
    type: DetectorTrainingRunner
    eval_interval: 50
    optimizer:
      type: AdamW
      params:
        lr: 1e-05
    scheduler:
      type: LRListScheduler
      params:
        milestones:
        - 90
        lr_list:
        - - 1e-06
    snapshots:
      max_snapshots: 5
      save_epochs: 50
      save_optimizer_state: false
  train_settings:
    batch_size: 32
    dataloader_workers: 32
    dataloader_pin_memory: true
    display_iters: 500
    epochs: 250
device: auto
metadata:
  project_path: /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23
  pose_config_path: 
    /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/dlc-models-pytorch/iteration-0/oppamousetrackerAug23-trainset90shuffle1/train/pose_cfg.yaml
  bodyparts:
  - nose
  - left_ear
  - right_ear
  - left_ear_tip
  - right_ear_tip
  - left_eye
  - right_eye
  - neck
  - mid_back
  - mouse_center
  - mid_backend
  - mid_backend2
  - mid_backend3
  - tail_base
  - tail1
  - tail2
  - tail3
  - tail4
  - tail5
  - left_shoulder
  - left_midside
  - left_hip
  - right_shoulder
  - right_midside
  - right_hip
  - tail_end
  - head_midpoint
  unique_bodyparts: []
  individuals:
  - individual1
  - individual2
  - individual3
  - individual4
  - individual5
  - individual6
  - individual7
  - individual8
  - individual9
  - individual10
  with_identity: false
method: td
model:
  backbone:
    type: HRNet
    model_name: hrnet_w32
    pretrained: false
    freeze_bn_stats: false
    freeze_bn_weights: false
    interpolate_branches: false
    increased_channel_count: false
  backbone_output_channels: 32
  heads:
    bodypart:
      type: HeatmapHead
      weight_init: normal
      predictor:
        type: HeatmapPredictor
        apply_sigmoid: false
        clip_scores: true
        location_refinement: false
        locref_std: 7.2801
      target_generator:
        type: HeatmapGaussianGenerator
        num_heatmaps: 27
        pos_dist_thresh: 17
        heatmap_mode: KEYPOINT
        generate_locref: false
        locref_std: 7.2801
      criterion:
        heatmap:
          type: WeightedMSECriterion
          weight: 1.0
      heatmap_config:
        channels:
        - 32
        - 27
        kernel_size:
        - 1
        strides:
        - 1
net_type: top_down_hrnet_w32
runner:
  type: PoseTrainingRunner
  key_metric: test.mAP
  key_metric_asc: true
  eval_interval: 10
  optimizer:
    type: AdamW
    params:
      lr: 1e-05
  scheduler:
    type: LRListScheduler
    params:
      lr_list:
      - - 1e-06
      - - 1e-07
      milestones:
      - 160
      - 190
  snapshots:
    max_snapshots: 5
    save_epochs: 25
    save_optimizer_state: false
train_settings:
  batch_size: 64
  dataloader_workers: 64
  dataloader_pin_memory: true
  display_iters: 500
  epochs: 200
  pretrained_weights:
  seed: 42
  weight_init:
    dataset: superanimal_topviewmouse
    with_decoder: true
    memory_replay: true
    conversion_array:
    - 0
    - 1
    - 2
    - 7
    - 9
    - 13
freeze_bn_stats: false

Anything else?

Code of Conduct

• I agree to follow this project's Code of Conduct

[n-poulsen]

Hi @Chap5732! I believe the issue comes from the fact that your dataset contains images that have different sizes ([3, 1688, 1695] at entry 0 and [3, 1688, 1726] at entry 1, so one image is 1695x1688 and another is 1726x1688), but there is no appropriate "collate" function to resize all of them to the same dimensions defined in your pytorch_config.yaml.

Having images of different sizes in a dataset is not at all a problem: you can either train with batch size 1, resize them all to the same size, or resize them when they are grouped into a batch so that all images in a batch have the same size. This last option is what I would do (and it's better explained how to do that exactly in the docs).

These SuperAnimal models are top-down (with an object detector to localize individuals, and a pose model to estimate pose). You only need to apply this collate configuration for the object detector, as the pose model will always receive images that are 256x256 pixels. You can set configure the collate function in your pytorch_config.yaml as follows:

detector:
  data:
    colormode: RGB
    inference:
      normalize_images: true
    train:
      hflip: true
      normalize_images: true
      collate:
        type: ResizeFromDataSizeCollate
        min_scale: 0.4
        max_scale: 1.0
        min_short_side: 128
        max_short_side: 1152
        multiple_of: 32
        to_square: false

I've noticed this collate configuration isn't added as a default for SuperAnimal models, so I'll update the code so that's the case.

Let me know if this fixes your bug or if you have other questions!

[Chap5732]

Thank you @n-poulsen ! After adding the collate function to my pytorch config file I successfully trained the model on my dataset!

But I encounter another error when I tried to evaluate the model :

DLC loaded in light mode; you cannot use any GUI (labeling, relabeling and standalone GUI)
Evaluating network...
100%|██████████████████████████████████████████████████████████████████████████| 338/338 [00:33<00:00, 10.19it/s]
100%|██████████████████████████████████████████████████████████████████████████| 338/338 [01:03<00:00,  5.35it/s]
Traceback (most recent call last):
  File "/ssd01/user_acc_data/oppa/deeplabcut/code/evaluation.py", line 16, in <module>
    evaluate_network(project_name, shuffle)
  File "/ssd01/user_acc_data/oppa/deeplabcut/code/evaluation.py", line 9, in evaluate_network
    deeplabcut.evaluate_network(config_path,
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/compat.py", line 476, in evaluate_network
    return evaluate_network(
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/evaluate.py", line 445, in evaluate_network
    evaluate_snapshot(
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/evaluate.py", line 239, in evaluate_snapshot
    results, predictions_for_split = evaluate(
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/evaluate.py", line 127, in evaluate
    pred["bodyparts"] = pred["bodyparts"][:, conversion_array]
IndexError: index 7 is out of bounds for axis 1 with size 6

And when I tried to skip the evaluation and perform the analyze_video step of the model on my test videos I encounter such error:

DLC loaded in light mode; you cannot use any GUI (labeling, relabeling and standalone GUI)
Analyzing all videos in folder: /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/analyze_videos/shuffle1/Approach/cropped with shuffle: 1
Analyzing videos with /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/dlc-models-pytorch/iteration-0/oppamousetrackerAug23-trainset90shuffle1/train/snapshot-200.pt
  -> Using detector /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/dlc-models-pytorch/iteration-0/oppamousetrackerAug23-trainset90shuffle1/train/snapshot-detector-250.pt
Starting to analyze /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/analyze_videos/shuffle1/Approach/cropped/DEMO1cropped.mp4
Video metadata: 
  Overall # of frames:    138
  Duration of video [s]:  4.60
  fps:                    30.0
  resolution:             w=1742, h=1672

Running Detector
100%|██████████████████████████████████████████████████████████████████████████| 138/138 [00:08<00:00, 15.34it/s]
Running Pose Prediction
100%|██████████████████████████████████████████████████████████████████████████| 138/138 [00:51<00:00,  2.69it/s]
Saving results in /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/analyze_videos/shuffle1/Approach/cropped/DEMO1croppedDLC_TopDownHrnetW32_oppamousetrackerAug23shuffle1_detector_250_snapshot_200.h5 and /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/analyze_videos/shuffle1/Approach/cropped/DEMO1croppedDLC_TopDownHrnetW32_oppamousetrackerAug23shuffle1_detector_250_snapshot_200_full.pickle
Traceback (most recent call last):
  File "/ssd01/user_acc_data/oppa/deeplabcut/code/analyze_videos.py", line 39, in <module>
    main()
  File "/ssd01/user_acc_data/oppa/deeplabcut/code/analyze_videos.py", line 36, in main
    analyze_videos_in_folder_or_file(config_path, input_path, shuffle)
  File "/ssd01/user_acc_data/oppa/deeplabcut/code/analyze_videos.py", line 23, in analyze_videos_in_folder_or_file
    deeplabcut.analyze_videos(config_path, video_files, save_as_csv=True, batchsize=64, shuffle=shuffle)
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/compat.py", line 856, in analyze_videos
    return analyze_videos(
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/analyze_videos.py", line 345, in analyze_videos
    df = create_df_from_prediction(
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/analyze_videos.py", line 432, in create_df_from_prediction
    df = pd.DataFrame(
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/pandas/core/frame.py", line 827, in __init__
    mgr = ndarray_to_mgr(
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/pandas/core/internals/construction.py", line 336, in ndarray_to_mgr
    _check_values_indices_shape_match(values, index, columns)
  File "/share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/pandas/core/internals/construction.py", line 420, in _check_values_indices_shape_match
    raise ValueError(f"Shape of passed values is {passed}, indices imply {implied}")
ValueError: Shape of passed values is (138, 810), indices imply (138, 180)

I noticed that the number of my labels, which is 6, can be multipled by 30 and get 180 and 810 can be got with 27 multiplying 30, which is the number of sa-tvm's labels. Does the error have something to do with the superanimal model config?

Here is my pytorch_config file:

Task:
scorer:
date:
multianimalproject:
identity:
project_path: 
  /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/dlc-models-pytorch/iteration-0/oppamousetrackerAug23-trainset90shuffle1/train
engine: tensorflow
video_sets:
bodyparts:
start:
stop:
numframes2pick:
skeleton: []
skeleton_color: black
pcutoff:
dotsize:
alphavalue:
colormap:
TrainingFraction:
iteration:
default_net_type:
default_augmenter:
snapshotindex:
detector_snapshotindex:
batch_size:
cropping:
x1:
x2:
y1:
y2:
corner2move2:
move2corner:
SuperAnimalConversionTables:
data:
  colormode: RGB
  inference:
    auto_padding:
      pad_width_divisor: 32
      pad_height_divisor: 32
    normalize_images: true
  train:
    affine:
      p: 0.5
      scaling:
      - 1.0
      - 1.0
      rotation: 30
      translation: 0
    gaussian_noise: 12.75
    normalize_images: true
    auto_padding:
      pad_width_divisor: 32
      pad_height_divisor: 32
detector:
  data:
    colormode: RGB
    inference:
      normalize_images: true
    train:
      hflip: true
      normalize_images: true
      collate:
        type: ResizeFromDataSizeCollate
        min_scale: 0.4
        max_scale: 1.0
        min_short_side: 128
        max_short_side: 1152
        multiple_of: 32
        to_square: false
  device: auto
  model:
    type: FasterRCNN
    variant: fasterrcnn_resnet50_fpn_v2
    box_score_thresh: 0.6
    pretrained: false
  runner:
    type: DetectorTrainingRunner
    eval_interval: 50
    optimizer:
      type: AdamW
      params:
        lr: 1e-05
    scheduler:
      type: LRListScheduler
      params:
        milestones:
        - 90
        lr_list:
        - - 1e-06
    snapshots:
      max_snapshots: 5
      save_epochs: 50
      save_optimizer_state: false
  train_settings:
    batch_size: 32
    dataloader_workers: 32
    dataloader_pin_memory: true
    display_iters: 500
    epochs: 250
device: auto
metadata:
  project_path: /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23
  pose_config_path: 
    /ssd01/user_acc_data/oppa/deeplabcut/projects/oppamousetracker-Oppa-2024-08-23/dlc-models-pytorch/iteration-0/oppamousetrackerAug23-trainset90shuffle1/train/pose_cfg.yaml
  bodyparts:
  - nose
  - left_ear
  - right_ear
  - left_ear_tip
  - right_ear_tip
  - left_eye
  - right_eye
  - neck
  - mid_back
  - mouse_center
  - mid_backend
  - mid_backend2
  - mid_backend3
  - tail_base
  - tail1
  - tail2
  - tail3
  - tail4
  - tail5
  - left_shoulder
  - left_midside
  - left_hip
  - right_shoulder
  - right_midside
  - right_hip
  - tail_end
  - head_midpoint
  unique_bodyparts: []
  individuals:
  - individual1
  - individual2
  - individual3
  - individual4
  - individual5
  - individual6
  - individual7
  - individual8
  - individual9
  - individual10
  with_identity: false
method: td
model:
  backbone:
    type: HRNet
    model_name: hrnet_w32
    pretrained: false
    freeze_bn_stats: false
    freeze_bn_weights: false
    interpolate_branches: false
    increased_channel_count: false
  backbone_output_channels: 32
  heads:
    bodypart:
      type: HeatmapHead
      weight_init: normal
      predictor:
        type: HeatmapPredictor
        apply_sigmoid: false
        clip_scores: true
        location_refinement: false
        locref_std: 7.2801
      target_generator:
        type: HeatmapGaussianGenerator
        num_heatmaps: 27
        pos_dist_thresh: 17
        heatmap_mode: KEYPOINT
        generate_locref: false
        locref_std: 7.2801
      criterion:
        heatmap:
          type: WeightedMSECriterion
          weight: 1.0
      heatmap_config:
        channels:
        - 32
        - 27
        kernel_size:
        - 1
        strides:
        - 1
net_type: top_down_hrnet_w32
runner:
  type: PoseTrainingRunner
  key_metric: test.mAP
  key_metric_asc: true
  eval_interval: 10
  optimizer:
    type: AdamW
    params:
      lr: 1e-05
  scheduler:
    type: LRListScheduler
    params:
      lr_list:
      - - 1e-06
      - - 1e-07
      milestones:
      - 160
      - 190
  snapshots:
    max_snapshots: 5
    save_epochs: 25
    save_optimizer_state: false
train_settings:
  batch_size: 64
  dataloader_workers: 32
  dataloader_pin_memory: true
  display_iters: 500
  epochs: 200
  pretrained_weights:
  seed: 42
  weight_init:
    dataset: superanimal_topviewmouse
    with_decoder: true
    memory_replay: true
    conversion_array:
    - 0
    - 1
    - 2
    - 7
    - 9
    - 13
freeze_bn_stats: false

[n-poulsen]

Thanks @Chap5732 - I'm looking into this

[n-poulsen]

@Chap5732 I wasn't able to replicate your issue - as you said, I believe this is due to the fact that somehow your project's labeled bodyparts (the 6) and the SuperAnimal bodyparts (the 27) are getting mixed up;

As the error is thrown from pred["bodyparts"] = pred["bodyparts"][:, conversion_array], it looks like predictions are only made for 6 bodyparts (while they should be made for all 27 when using memory replay).

Could you confirm that you're using the latest version of the code?

[Chap5732]

@Chap5732 I wasn't able to replicate your issue - as you said, I believe this is due to the fact that somehow your project's labeled bodyparts (the 6) and the SuperAnimal bodyparts (the 27) are getting mixed up;

As the error is thrown from pred["bodyparts"] = pred["bodyparts"][:, conversion_array], it looks like predictions are only made for 6 bodyparts (while they should be made for all 27 when using memory replay).

Could you confirm that you're using the latest version of the code?

Hi, thank you for your quick response!

My senior told me to add this line at L353
pred_bodyparts = pred_bodyparts[:, :, [0, 1, 2, 7, 9, 13], :]
in the /share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/analyze_videos.py source code file

Cuz I am not good at coding so I'll just paste the changed source code here.

#
# DeepLabCut Toolbox (deeplabcut.org)
# © A. & M.W. Mathis Labs
# https://github.com/DeepLabCut/DeepLabCut
#
# Please see AUTHORS for contributors.
# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS
#
# Licensed under GNU Lesser General Public License v3.0
#
from __future__ import annotations

import copy
import logging
import pickle
import time
from pathlib import Path
from typing import Any

import albumentations as A
import numpy as np
import pandas as pd
from tqdm import tqdm

from deeplabcut.core.engine import Engine
from deeplabcut.pose_estimation_pytorch.apis.convert_detections_to_tracklets import (
    convert_detections2tracklets,
)
from deeplabcut.pose_estimation_pytorch.apis.utils import (
    get_model_snapshots,
    get_inference_runners,
    get_scorer_name,
    get_scorer_uid,
    list_videos_in_folder,
    parse_snapshot_index_for_analysis,
)
from deeplabcut.pose_estimation_pytorch.post_processing.identity import assign_identity
from deeplabcut.pose_estimation_pytorch.runners import InferenceRunner
from deeplabcut.pose_estimation_pytorch.task import Task
from deeplabcut.refine_training_dataset.stitch import stitch_tracklets
from deeplabcut.utils import auxfun_multianimal, auxiliaryfunctions, VideoReader


class VideoIterator(VideoReader):
    """A class to iterate over videos, with possible added context"""

    def __init__(
        self, video_path: str | Path, context: list[dict[str, Any]] | None = None
    ) -> None:
        super().__init__(str(video_path))
        self._context = context
        self._index = 0

    def get_context(self) -> list[dict[str, Any]] | None:
        if self._context is None:
            return None

        return copy.deepcopy(self._context)

    def set_context(self, context: list[dict[str, Any]] | None) -> None:
        if context is None:
            self._context = None
            return

        self._context = copy.deepcopy(context)

    def __iter__(self):
        return self

    def __next__(self) -> np.ndarray | tuple[str, dict[str, Any]]:
        frame = self.read_frame()
        if frame is None:
            self._index = 0
            self.reset()
            raise StopIteration

        # Otherwise ValueError: At least one stride in the given numpy array is negative,
        # and tensors with negative strides are not currently supported. (You can probably
        # work around this by making a copy of your array  with array.copy().)
        frame = frame.copy()
        if self._context is None:
            self._index += 1
            return frame

        context = copy.deepcopy(self._context[self._index])
        self._index += 1
        return frame, context


def video_inference(
    video_path: str | Path,
    task: Task,
    pose_runner: InferenceRunner,
    detector_runner: InferenceRunner | None = None,
    with_identity: bool = False,
    return_video_metadata: bool = False,
) -> list[dict[str, np.ndarray]]:
    """Runs inference on a video"""
    video = VideoIterator(str(video_path))
    n_frames = video.get_n_frames()
    vid_w, vid_h = video.dimensions
    print(f"Starting to analyze {video_path}")
    print(
        f"Video metadata: \n"
        f"  Overall # of frames:    {n_frames}\n"
        f"  Duration of video [s]:  {n_frames / max(1, video.fps):.2f}\n"
        f"  fps:                    {video.fps}\n"
        f"  resolution:             w={vid_w}, h={vid_h}\n"
    )
    video_metadata = {
        "n_frames": n_frames,
        "fps": video.fps,
        "resolution": (vid_w, vid_h),
    }

    if task == Task.TOP_DOWN:
        # Get bounding boxes for context
        if detector_runner is None:
            raise ValueError("Must use a detector for top-down video analysis")

        print(f"Running detector with batch size {detector_runner.batch_size}")
        bbox_predictions = detector_runner.inference(images=tqdm(video))
        video.set_context(bbox_predictions)

    print(f"Running pose prediction with batch size {pose_runner.batch_size}")
    predictions = pose_runner.inference(images=tqdm(video))

    if with_identity:
        bodypart_predictions = assign_identity(
            [p["bodyparts"] for p in predictions],
            [p["identity_scores"] for p in predictions],
        )
        for i, p_with_id in enumerate(bodypart_predictions):
            predictions[i]["bodyparts"] = p_with_id

    if len(predictions) != n_frames:
        tip_url = "https://deeplabcut.github.io/DeepLabCut/docs/recipes/io.html"
        header = "#tips-on-video-re-encoding-and-preprocessing"
        logging.warning(
            f"The video metadata indicates that there {n_frames} in the video, but "
            f"only {len(predictions)} were able to be processed. This can happen if "
            "the video is corrupted. You can try to fix the issue by re-encoding your "
            f"video (tips on how to do that: {tip_url}{header})"
        )

    if return_video_metadata:
        return predictions, video_metadata

    return predictions


def analyze_videos(
    config: str,
    videos: str | list[str],
    videotype: str | None = None,
    shuffle: int = 1,
    trainingsetindex: int = 0,
    save_as_csv: bool = False,
    snapshot_index: int | str | None = None,
    detector_snapshot_index: int | str | None = None,
    device: str | None = None,
    destfolder: str | None = None,
    batch_size: int | None = None,
    detector_batch_size: int | None = None,
    modelprefix: str = "",
    transform: A.Compose | None = None,
    auto_track: bool | None = True,
    identity_only: bool | None = False,
    overwrite: bool = False,
) -> str:
    """Makes prediction based on a trained network.

    # TODO:
        - other options missing options such as shelve
        - pass detector path or detector runner

    The index of the trained network is specified by parameters in the config file
    (in particular the variable 'snapshot_index').

    Args:
        config: full path of the config.yaml file for the project
        videos: a str (or list of strings) containing the full paths to videos for
            analysis or a path to the directory, where all the videos with same
            extension are stored.
        videotype: checks for the extension of the video in case the input to the video
            is a directory. Only videos with this extension are analyzed. If left
            unspecified, keeps videos with extensions ('avi', 'mp4', 'mov', 'mpeg', 'mkv').
        shuffle: An integer specifying the shuffle index of the training dataset used for
            training the network.
        trainingsetindex: Integer specifying which TrainingsetFraction to use.
        save_as_csv: Saves the predictions in a .csv file.
        device: the device to use for video analysis
        destfolder: specifies the destination folder for analysis data. If ``None``,
            the path of the video is used. Note that for subsequent analysis this
            folder also needs to be passed
        snapshot_index: index (starting at 0) of the snapshot to use to analyze the
            videos. To evaluate the last one, use -1. For example if we have
                - snapshot-0.pt
                - snapshot-50.pt
                - snapshot-100.pt
                - snapshot-best.pt
            and we want to evaluate snapshot-50.pt, snapshotindex should be 1. If None,
            the snapshot index is loaded from the project configuration.
        detector_snapshot_index: (only for top-down models) index of the detector
            snapshot to use, used in the same way as ``snapshot_index``
        modelprefix: directory containing the deeplabcut models to use when evaluating
            the network. By default, they are assumed to exist in the project folder.
        batch_size: the batch size to use for inference. Takes the value from the
            project config as a default.
        detector_batch_size: the batch size to use for detector inference. Takes the
            value from the project config as a default.
        transform: Optional custom transforms to apply to the video
        overwrite: Overwrite any existing videos
        auto_track: By default, tracking and stitching are automatically performed,
            producing the final h5 data file. This is equivalent to the behavior for
            single-animal projects.

            If ``False``, one must run ``convert_detections2tracklets`` and
            ``stitch_tracklets`` afterwards, in order to obtain the h5 file.
        identity_only: sub-call for auto_track. If ``True`` and animal identity was
            learned by the model, assembly and tracking rely exclusively on identity
            prediction.

    Returns:
        The scorer used to analyze the videos
    """
    # Create the output folder
    _validate_destfolder(destfolder)

    # Load the project configuration
    cfg = auxiliaryfunctions.read_config(config)
    project_path = Path(cfg["project_path"])
    train_fraction = cfg["TrainingFraction"][trainingsetindex]
    model_folder = project_path / auxiliaryfunctions.get_model_folder(
        train_fraction,
        shuffle,
        cfg,
        modelprefix=modelprefix,
        engine=Engine.PYTORCH,
    )
    train_folder = model_folder / "train"

    # Read the inference configuration, load the model
    model_cfg_path = train_folder / Engine.PYTORCH.pose_cfg_name
    model_cfg = auxiliaryfunctions.read_plainconfig(model_cfg_path)
    pose_task = Task(model_cfg["method"])

    pose_cfg_path = model_folder / "test" / "pose_cfg.yaml"
    pose_cfg = auxiliaryfunctions.read_plainconfig(pose_cfg_path)

    snapshot_index, detector_snapshot_index = parse_snapshot_index_for_analysis(
        cfg, model_cfg, snapshot_index, detector_snapshot_index,
    )

    # Get general project parameters
    bodyparts = model_cfg["metadata"]["bodyparts"]
    unique_bodyparts = model_cfg["metadata"]["unique_bodyparts"]
    individuals = model_cfg["metadata"]["individuals"]
    with_identity = model_cfg["metadata"]["with_identity"]
    max_num_animals = len(individuals)

    if device is not None:
        model_cfg["device"] = device

    if batch_size is None:
        batch_size = cfg.get("batch_size", 1)

    snapshot = get_model_snapshots(snapshot_index, train_folder, pose_task)[0]
    print(f"Analyzing videos with {snapshot.path}")
    detector_path, detector_snapshot = None, None
    if pose_task == Task.TOP_DOWN:
        if detector_snapshot_index is None:
            raise ValueError(
                "Cannot run videos analysis for top-down models without a detector "
                "snapshot! Please specify your desired detector_snapshotindex in your "
                "project's configuration file."
            )

        if detector_batch_size is None:
            detector_batch_size = cfg.get("detector_batch_size", 1)

        detector_snapshot = get_model_snapshots(
            detector_snapshot_index, train_folder, Task.DETECT
        )[0]
        detector_path = detector_snapshot.path
        print(f"  -> Using detector {detector_path}")

    dlc_scorer = get_scorer_name(
        cfg,
        shuffle,
        train_fraction,
        snapshot_uid=get_scorer_uid(snapshot, detector_snapshot),
        modelprefix=modelprefix,
    )
    pose_runner, detector_runner = get_inference_runners(
        model_config=model_cfg,
        snapshot_path=snapshot.path,
        max_individuals=max_num_animals,
        num_bodyparts=len(bodyparts),
        num_unique_bodyparts=len(unique_bodyparts),
        batch_size=batch_size,
        with_identity=with_identity,
        transform=transform,
        detector_batch_size=detector_batch_size,
        detector_path=detector_path,
        detector_transform=None,
    )

    # Reading video and init variables
    videos = list_videos_in_folder(videos, videotype)
    results = []
    for video in videos:
        if destfolder is None:
            output_path = video.parent
        else:
            output_path = Path(destfolder)

        output_prefix = video.stem + dlc_scorer
        output_h5 = output_path / f"{output_prefix}.h5"
        output_pkl = output_path / f"{output_prefix}_full.pickle"

        if not overwrite and output_pkl.exists():
            print(f"Video {video} already analyzed at {output_pkl}!")
        else:
            runtime = [time.time()]
            predictions = video_inference(
                video_path=video,
                pose_runner=pose_runner,
                task=pose_task,
                detector_runner=detector_runner,
            )
            runtime.append(time.time())
            metadata = _generate_metadata(
                cfg=cfg,
                pytorch_config=model_cfg,
                dlc_scorer=dlc_scorer,
                train_fraction=train_fraction,
                batch_size=batch_size,
                runtime=(runtime[0], runtime[1]),
                video=VideoReader(str(video)),
            )
            output_data = _generate_output_data(pose_cfg, predictions)
            _ = auxfun_multianimal.SaveFullMultiAnimalData(
                output_data, metadata, str(output_h5)
            )

            pred_bodyparts = np.stack([p["bodyparts"][..., :3] for p in predictions])
            pred_unique_bodyparts = None
            if len(predictions) > 0 and "unique_bodyparts" in predictions[0]:
                pred_unique_bodyparts = np.stack(
                    [p["unique_bodyparts"] for p in predictions]
                )
            pred_bodyparts = pred_bodyparts[:, :, [0, 1, 2, 7, 9, 13], :]
            df = create_df_from_prediction(
                pred_bodyparts=pred_bodyparts,
                pred_unique_bodyparts=pred_unique_bodyparts,
                cfg=cfg,
                dlc_scorer=dlc_scorer,
                output_path=output_path,
                output_prefix=output_prefix,
                save_as_csv=save_as_csv,
            )
            results.append((str(video), df))

            if cfg["multianimalproject"] and len(individuals) > 1:
                pred_bodypart_ids = None
                if with_identity:
                    # reshape from (num_assemblies, num_bpts, num_individuals)
                    # to (num_assemblies, num_bpts) by taking the maximum
                    # likelihood individual for each bodypart
                    pred_bodypart_ids = np.stack(
                        [np.argmax(p["identity_scores"], axis=2) for p in predictions]
                    )

                _save_assemblies(
                    output_path,
                    output_prefix,
                    pred_bodyparts,
                    pred_bodypart_ids,
                    pred_unique_bodyparts,
                    with_identity,
                )
                if auto_track:
                    convert_detections2tracklets(
                        config=config,
                        videos=str(video),
                        videotype=videotype,
                        shuffle=shuffle,
                        trainingsetindex=trainingsetindex,
                        overwrite=False,
                        identity_only=identity_only,
                        destfolder=str(output_path),
                    )
                    stitch_tracklets(
                        config,
                        [str(video)],
                        videotype,
                        shuffle,
                        trainingsetindex,
                        destfolder=str(output_path),
                    )

    print(
        "The videos are analyzed. Now your research can truly start!\n"
        "You can create labeled videos with 'create_labeled_video'.\n"
        "If the tracking is not satisfactory for some videos, consider expanding the "
        "training set. You can use the function 'extract_outlier_frames' to extract a "
        "few representative outlier frames.\n"
    )

    return dlc_scorer


def create_df_from_prediction(
    pred_bodyparts: np.ndarray,
    pred_unique_bodyparts: np.ndarray | None,
    dlc_scorer: str,
    cfg: dict,
    output_path: str | Path,
    output_prefix: str | Path,
    save_as_csv: bool = False,
) -> pd.DataFrame:
    output_h5 = Path(output_path) / f"{output_prefix}.h5"
    output_pkl = Path(output_path) / f"{output_prefix}_full.pickle"

    print(f"Saving results in {output_h5} and {output_pkl}")
    cols = [
        [dlc_scorer],
        list(auxiliaryfunctions.get_bodyparts(cfg)),
        ["x", "y", "likelihood"],
    ]
    cols_names = ["scorer", "bodyparts", "coords"]
    individuals = cfg.get("individuals", ["animal"])
    n_individuals = len(individuals)
    if n_individuals > 1:
        cols.insert(1, individuals)
        cols_names.insert(1, "individuals")

    results_df_index = pd.MultiIndex.from_product(cols, names=cols_names)
    pred_bodyparts = pred_bodyparts[:, :n_individuals]
    df = pd.DataFrame(
        pred_bodyparts.reshape((len(pred_bodyparts), -1)),
        columns=results_df_index,
        index=range(len(pred_bodyparts)),
    )
    if pred_unique_bodyparts is not None:
        coordinate_labels_unique = ["x", "y", "likelihood"]
        if n_individuals > 1:
            results_unique_df_index = pd.MultiIndex.from_product(
                [
                    [dlc_scorer],
                    ['single'],
                    auxiliaryfunctions.get_unique_bodyparts(cfg),
                    coordinate_labels_unique,
                ],
                names=["scorer", "individuals", "bodyparts", "coords"],
            )
        else:
            results_unique_df_index = pd.MultiIndex.from_product(
                [
                    [dlc_scorer],
                    auxiliaryfunctions.get_unique_bodyparts(cfg),
                    coordinate_labels_unique,
                ],
                names=["scorer", "bodyparts", "coords"],
            )
        df_u = pd.DataFrame(
            pred_unique_bodyparts.reshape((len(pred_unique_bodyparts), -1)),
            columns=results_unique_df_index,
            index=range(len(pred_unique_bodyparts)),
        )
        df = df.join(df_u, how="outer")

    df.to_hdf(output_h5, key="df_with_missing", format="table", mode="w")
    if save_as_csv:
        df.to_csv(output_h5.with_suffix(".csv"))
    return df


def _save_assemblies(
    output_path: Path,
    output_prefix: str,
    pred_bodyparts: np.ndarray,
    pred_bodypart_ids: np.ndarray,
    pred_unique_bodyparts: np.ndarray,
    with_identity: bool,
) -> None:
    output_ass = output_path / f"{output_prefix}_assemblies.pickle"
    assemblies = {}
    for i, bpt in enumerate(pred_bodyparts):
        if with_identity:
            extra_column = np.expand_dims(pred_bodypart_ids[i], axis=-1)
        else:
            extra_column = np.full(
                (bpt.shape[0], bpt.shape[1], 1),
                -1.0,
                dtype=np.float32,
            )
        ass = np.concatenate((bpt, extra_column), axis=-1)
        assemblies[i] = ass

    if pred_unique_bodyparts is not None:
        assemblies["single"] = {}
        for i, unique_bpt in enumerate(pred_unique_bodyparts):
            extra_column = np.full((unique_bpt.shape[1], 1), -1.0, dtype=np.float32)
            ass = np.concatenate((unique_bpt[0], extra_column), axis=-1)
            assemblies["single"][i] = ass

    with open(output_ass, "wb") as handle:
        pickle.dump(assemblies, handle, protocol=pickle.HIGHEST_PROTOCOL)


def _validate_destfolder(destfolder: str | None) -> None:
    """Checks that the destfolder for video analysis is valid"""
    if destfolder is not None and destfolder != "":
        output_folder = Path(destfolder)
        if not output_folder.exists():
            print(f"Creating the output folder {output_folder}")
            output_folder.mkdir(parents=True)

        assert Path(
            output_folder
        ).is_dir(), f"Output folder must be a directory: you passed '{output_folder}'"


def _generate_metadata(
    cfg: dict,
    pytorch_config: dict,
    dlc_scorer: str,
    train_fraction: int,
    batch_size: int,
    runtime: tuple[float, float],
    video: VideoReader,
) -> dict:
    w, h = video.dimensions
    cropping = cfg.get("cropping", False)
    if cropping:
        cropping_parameters = [cfg["x1"], cfg["x2"], cfg["y1"], cfg["y2"]]
    else:
        cropping_parameters = [0, w, 0, h]

    metadata = {
        "start": runtime[0],
        "stop": runtime[1],
        "run_duration": runtime[1] - runtime[0],
        "Scorer": dlc_scorer,
        "pytorch-config": pytorch_config,
        "fps": video.fps,
        "batch_size": batch_size,
        "frame_dimensions": (w, h),
        "nframes": video.get_n_frames(),
        "iteration (active-learning)": cfg["iteration"],
        "training set fraction": train_fraction,
        "cropping": cropping,
        "cropping_parameters": cropping_parameters,
    }
    return {"data": metadata}


def _generate_output_data(
    pose_config: dict,
    predictions: list[dict[str, np.ndarray]],
) -> dict:
    output = {
        "metadata": {
            "nms radius": pose_config.get("nmsradius"),
            "minimal confidence": pose_config.get("minconfidence"),
            "sigma": pose_config.get("sigma", 1),
            "PAFgraph": pose_config.get("partaffinityfield_graph"),
            "PAFinds": pose_config.get(
                "paf_best",
                np.arange(len(pose_config.get("partaffinityfield_graph", []))),
            ),
            "all_joints": [[i] for i in range(len(pose_config["all_joints"]))],
            "all_joints_names": [
                pose_config["all_joints_names"][i]
                for i in range(len(pose_config["all_joints"]))
            ],
            "nframes": len(predictions),
        }
    }

    str_width = int(np.ceil(np.log10(len(predictions))))
    for frame_num, frame_predictions in enumerate(predictions):
        # TODO: Do we want to keep the same format as in the TensorFlow version?
        #  On the one hand, it's "more" backwards compatible.
        #  On the other, might as well simplify the code. These files should only be loaded
        #    by the PyTorch version, and only predictions made by PyTorch models should be
        #    loaded using them

        key = "frame" + str(frame_num).zfill(str_width)
        bodyparts = frame_predictions[
            "bodyparts"
        ]  # shape (num_assemblies, num_bpts, 3)
        bodyparts = bodyparts.transpose(
            (1, 0, 2)
        )  # shape (num_bpts, num_assemblies, 3)
        coordinates = [bpt[:, :2] for bpt in bodyparts]
        scores = [bpt[:, 2:] for bpt in bodyparts]

        # full pickle has bodyparts and unique bodyparts in same array
        if "unique_bodyparts" in frame_predictions:
            unique_bpts = frame_predictions["unique_bodyparts"].transpose((1, 0, 2))
            coordinates += [bpt[:, :2] for bpt in unique_bpts]
            scores += [bpt[:, 2:] for bpt in unique_bpts]

        output[key] = {
            "coordinates": (coordinates,),
            "confidence": scores,
            "costs": None,
        }

        if "identity_scores" in frame_predictions:
            # Reshape id scores from (num_assemblies, num_bpts, num_individuals)
            # to the original DLC full pickle format: (num_bpts, num_assem, num_ind)
            id_scores = frame_predictions["identity_scores"]
            id_scores = id_scores.transpose((1, 0, 2))
            output[key]["identity"] = [bpt_id_scores for bpt_id_scores in id_scores]

    return output

And after I did the changing the analyze_video function successfully analyzed my test videos.

Hope this helps :)

[Chap5732]

And about the evaluation step,

I replace the L127 in /share/user_data/oppa/.conda/envs/dlc-oppa/lib/python3.10/site-packages/deeplabcut/pose_estimation_pytorch/apis/evaluate.py with

if pred["bodyparts"].shape[1] > 6:
    pred["bodyparts"] = pred["bodyparts"][:, conversion_array, :]

And then I successfully finish the evaluation.

Changed source code

#
# DeepLabCut Toolbox (deeplabcut.org)
# © A. & M.W. Mathis Labs
# https://github.com/DeepLabCut/DeepLabCut
#
# Please see AUTHORS for contributors.
# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS
#
# Licensed under GNU Lesser General Public License v3.0
#
from __future__ import annotations

import argparse
from pathlib import Path
from typing import Iterable

import albumentations as A
import numpy as np
import pandas as pd
from tqdm import tqdm

import deeplabcut.core.metrics as metrics
from deeplabcut.pose_estimation_pytorch import utils
from deeplabcut.pose_estimation_pytorch.apis.utils import (
    build_predictions_dataframe,
    ensure_multianimal_df_format,
    get_inference_runners,
    get_model_snapshots,
    get_scorer_name,
    get_scorer_uid,
)
from deeplabcut.pose_estimation_pytorch.data import DLCLoader, Loader
from deeplabcut.pose_estimation_pytorch.data.dataset import PoseDatasetParameters
from deeplabcut.pose_estimation_pytorch.runners import InferenceRunner
from deeplabcut.pose_estimation_pytorch.runners.snapshots import Snapshot
from deeplabcut.pose_estimation_pytorch.task import Task
from deeplabcut.utils import auxiliaryfunctions
from deeplabcut.utils.visualization import plot_evaluation_results


def predict(
    pose_task: Task,
    pose_runner: InferenceRunner,
    loader: Loader,
    mode: str,
    detector_runner: InferenceRunner | None = None,
) -> dict[str, dict[str, np.ndarray]]:
    """Predicts poses on data contained in a loader

    Args:
        pose_task: Whether the model is a top-down or bottom-up model
        pose_runner: The runner to use for pose estimation
        loader: The loader containing the data to predict poses on
        mode: {"train", "test"} The mode to predict on
        detector_runner: If the task is "TD", a detector runner can be given to detect
            individuals in the images. If no detector is given, ground truth bounding
            boxes will be used to crop individuals before pose estimation

    Returns:
        The paths of images for which predictions were computed mapping to the
        different predictions made by each model head
    """
    image_paths = loader.image_filenames(mode)
    context = None

    if pose_task == Task.TOP_DOWN:
        # Get bounding boxes for context
        if detector_runner is not None:
            bbox_predictions = detector_runner.inference(images=tqdm(image_paths))
            context = bbox_predictions
        else:
            ground_truth_bboxes = loader.ground_truth_bboxes(mode=mode)
            context = [{"bboxes": ground_truth_bboxes[image]} for image in image_paths]

    images_with_context = image_paths
    if context is not None:
        if len(context) != len(image_paths):
            raise ValueError(
                f"Missing context for some images: {len(context)} != {len(image_paths)}"
            )
        images_with_context = list(zip(image_paths, context))

    predictions = pose_runner.inference(images=tqdm(images_with_context))
    return {
        image_path: image_predictions
        for image_path, image_predictions in zip(image_paths, predictions)
    }


def evaluate(
    pose_task: Task,
    pose_runner: InferenceRunner,
    loader: Loader,
    mode: str,
    detector_runner: InferenceRunner | None = None,
    pcutoff: float = 1,
) -> tuple[dict[str, float], dict[str, dict[str, np.ndarray]]]:
    """
    Args:
        pose_task: Whether to run top-down or bottom-up
        pose_runner: The runner for pose estimation
        loader: The loader containing the data to evaluate
        mode: Either 'train' or 'test'
        detector_runner: If task == 'TD', a detector can be given to compute bounding
            boxes for pose estimation. If no detector is given, ground truth bounding
            boxes are used
        pcutoff: The p-cutoff to use for evaluation

    Returns:
        A dict containing the evaluation results
        A dict mapping the paths of images for which predictions were computed to the
            different predictions made by each model head
    """
    parameters = loader.get_dataset_parameters()
    predictions = predict(
        pose_task=pose_task,
        pose_runner=pose_runner,
        loader=loader,
        mode=mode,
        detector_runner=detector_runner,
    )
    if "weight_init" in loader.model_cfg["train_settings"]:
        weight_init_cfg = loader.model_cfg["train_settings"]["weight_init"]
        if weight_init_cfg["memory_replay"]:
            conversion_array = weight_init_cfg["conversion_array"]
            for filename, pred in predictions.items():
                if pred["bodyparts"].shape[1] > 6:
                    pred["bodyparts"] = pred["bodyparts"][:, conversion_array, :]
                

    poses = {filename: pred["bodyparts"] for filename, pred in predictions.items()}

    gt_keypoints = loader.ground_truth_keypoints(mode)
    unique_poses = None
    gt_unique_keypoints = None
    if parameters.num_unique_bpts > 1:
        unique_poses = {
            filename: pred["unique_bodyparts"] for filename, pred in predictions.items()
        }
        gt_unique_keypoints = loader.ground_truth_keypoints(mode, unique_bodypart=True)

    results = metrics.compute_metrics(
        gt_keypoints,
        poses,
        single_animal=parameters.max_num_animals == 1,
        pcutoff=pcutoff,
        unique_bodypart_poses=unique_poses,
        unique_bodypart_gt=gt_unique_keypoints,
    )

    if loader.model_cfg["metadata"]["with_identity"]:
        pred_id_scores = {
            filename: pred["identity_scores"] for filename, pred in predictions.items()
        }
        id_scores = metrics.compute_identity_scores(
            individuals=parameters.individuals,
            bodyparts=parameters.bodyparts,
            predictions=poses,
            identity_scores=pred_id_scores,
            ground_truth=gt_keypoints,
        )
        for name, score in id_scores.items():
            results[f"id_head_{name}"] = score

    # Updating poses to be aligned and padded
    for image, pose in poses.items():
        predictions[image]["bodyparts"] = pose

    return results, predictions


def evaluate_snapshot(
    cfg: dict,
    loader: DLCLoader,
    snapshot: Snapshot,
    scorer: str,
    transform: A.Compose | None = None,
    plotting: bool | str = False,
    show_errors: bool = True,
    detector_snapshot: Snapshot | None = None,
) -> pd.DataFrame:
    """Evaluates a snapshot.
    The evaluation results are stored in the .h5 and .csv file under the subdirectory
    'evaluation_results'.

    Args:
        cfg: the content of the project's config file
        loader: the loader for the shuffle to evaluate
        snapshot: the snapshot to evaluate
        scorer: the scorer name to use for the snapshot
        transform: transformation pipeline for evaluation
            ** Should normalise the data the same way it was normalised during training **
        plotting: Plots the predictions on the train and test images. If provided it must
            be either ``True``, ``False``, ``"bodypart"``, or ``"individual"``. Setting
            to ``True`` defaults as ``"bodypart"`` for multi-animal projects.
        show_errors: whether to compare predictions and ground truth
        detector_snapshot: Only for TD models. If defined, evaluation metrics are
            computed using the detections made by this snapshot
    """
    pose_task = Task(loader.model_cfg.get("method", "bu"))
    parameters = loader.get_dataset_parameters()

    if "weight_init" in loader.model_cfg["train_settings"]:
        weight_init_cfg = loader.model_cfg["train_settings"]["weight_init"]
        if weight_init_cfg["memory_replay"]:
            conversion_array = weight_init_cfg["conversion_array"]
            bodyparts = list(np.array(parameters.bodyparts)[conversion_array])
            parameters = PoseDatasetParameters(
                bodyparts, parameters.unique_bpts, parameters.individuals
            )

    pcutoff = cfg.get("pcutoff")

    detector_path = None
    if detector_snapshot is not None:
        detector_path = detector_snapshot.path

    pose_runner, detector_runner = get_inference_runners(
        model_config=loader.model_cfg,
        snapshot_path=snapshot.path,
        max_individuals=parameters.max_num_animals,
        num_bodyparts=parameters.num_joints,
        num_unique_bodyparts=parameters.num_unique_bpts,
        with_identity=loader.model_cfg["metadata"]["with_identity"],
        transform=transform,
        detector_path=detector_path,
        detector_transform=None,
    )

    predictions = {}
    scores = {
        "%Training dataset": loader.train_fraction,
        "Shuffle number": loader.shuffle,
        "Training epochs": snapshot.epochs,
        "Detector epochs (TD only)": (
            -1 if detector_snapshot is None else detector_snapshot.epochs
        ),
        "pcutoff": pcutoff,
    }
    for split in ["train", "test"]:
        results, predictions_for_split = evaluate(
            pose_task=pose_task,
            pose_runner=pose_runner,
            loader=loader,
            mode=split,
            pcutoff=pcutoff,
            detector_runner=detector_runner,
        )
        df_split_predictions = build_predictions_dataframe(
            scorer=scorer,
            predictions=predictions_for_split,
            parameters=parameters,
            image_name_to_index=image_to_dlc_df_index,
        )
        predictions[split] = df_split_predictions
        for k, v in results.items():
            scores[f"{split} {k}"] = round(v, 2)

    results_filename = f"{scorer}.h5"
    df_predictions = pd.concat(predictions.values(), axis=0)
    df_predictions = df_predictions.reindex(loader.df.index)
    output_filename = loader.evaluation_folder / results_filename
    output_filename.parent.mkdir(parents=True, exist_ok=True)
    df_predictions.to_hdf(output_filename, key="df_with_missing")

    df_scores = pd.DataFrame([scores]).set_index(
        [
            "%Training dataset",
            "Shuffle number",
            "Training epochs",
            "Detector epochs (TD only)",
            "pcutoff",
        ]
    )
    scores_filepath = output_filename.with_suffix(".csv")
    scores_filepath = scores_filepath.with_stem(scores_filepath.stem + "-results")
    save_evaluation_results(df_scores, scores_filepath, show_errors, pcutoff)

    if plotting:
        folder_name = f"LabeledImages_{scorer}"
        folder_path = loader.evaluation_folder / folder_name
        folder_path.mkdir(parents=True, exist_ok=True)
        if isinstance(plotting, str):
            plot_mode = plotting
        else:
            plot_mode = "bodypart"

        df_ground_truth = ensure_multianimal_df_format(loader.df)
        for mode in ["train", "test"]:
            df_combined = predictions[mode].merge(
                df_ground_truth, left_index=True, right_index=True
            )
            unique_bodyparts = loader.get_dataset_parameters().unique_bpts
            plot_evaluation_results(
                df_combined=df_combined,
                project_root=cfg["project_path"],
                scorer=cfg["scorer"],
                model_name=scorer,
                output_folder=str(folder_path),
                in_train_set=mode == "train",
                plot_unique_bodyparts=len(unique_bodyparts) > 0,
                mode=plot_mode,
                colormap=cfg["colormap"],
                dot_size=cfg["dotsize"],
                alpha_value=cfg["alphavalue"],
                p_cutoff=cfg["pcutoff"],
            )

    return df_predictions


def evaluate_network(
    config: str,
    shuffles: Iterable[int] = (1,),
    trainingsetindex: int | str = 0,
    snapshotindex: int | str | None = None,
    device: str | None = None,
    plotting: bool | str = False,
    show_errors: bool = True,
    transform: A.Compose = None,
    modelprefix: str = "",
    detector_snapshot_index: int | None = None,
) -> None:
    """Evaluates a snapshot.

    The evaluation results are stored in the .h5 and .csv file under the subdirectory
    'evaluation_results'.

    Args:
        config: path to the project's config file
        shuffles: Iterable of integers specifying the shuffle indices to evaluate.
        trainingsetindex: Integer specifying which training set fraction to use.
            Evaluates all fractions if set to "all"
        snapshotindex: index (starting at 0) of the snapshot we want to load. To
            evaluate the last one, use -1. To evaluate all snapshots, use "all". For
            example if we have 3 models saved
                - snapshot-0.pt
                - snapshot-50.pt
                - snapshot-100.pt
            and we want to evaluate snapshot-50.pt, snapshotindex should be 1. If None,
            the snapshotindex is loaded from the project configuration.
        device: the device to run evaluation on
        plotting: Plots the predictions on the train and test images. If provided it must
            be either ``True``, ``False``, ``"bodypart"``, or ``"individual"``. Setting
            to ``True`` defaults as ``"bodypart"`` for multi-animal projects.
        show_errors: display train and test errors.
        transform: transformation pipeline for evaluation
            ** Should normalise the data the same way it was normalised during training **
        modelprefix: directory containing the deeplabcut models to use when evaluating
            the network. By default, they are assumed to exist in the project folder.
        detector_snapshot_index: Only for TD models. If defined, uses the detector with
            the given index for pose estimation.

    Examples:
        If you want to evaluate on shuffle 1 without plotting predictions.

        >>> import deeplabcut
        >>> deeplabcut.evaluate_network(
        >>>     '/analysis/project/reaching-task/config.yaml', shuffles=[1],
        >>> )

        If you want to evaluate shuffles 0 and 1 and plot the predictions.

        >>> deeplabcut.evaluate_network(
        >>>     '/analysis/project/reaching-task/config.yaml',
        >>>     shuffles=[0, 1],
        >>>     plotting=True,
        >>> )

        If you want to plot assemblies for a maDLC project

        >>> deeplabcut.evaluate_network(
        >>>     '/analysis/project/reaching-task/config.yaml',
        >>>     shuffles=[1],
        >>>     plotting="individual",
        >>> )
    """
    cfg = auxiliaryfunctions.read_config(config)

    if isinstance(trainingsetindex, int):
        train_set_indices = [trainingsetindex]
    elif isinstance(trainingsetindex, str) and trainingsetindex.lower() == "all":
        train_set_indices = list(range(len(cfg["TrainingFraction"])))
    else:
        raise ValueError(f"Invalid trainingsetindex: {trainingsetindex}")

    if snapshotindex is None:
        snapshotindex = cfg["snapshotindex"]

    if detector_snapshot_index is None:
        detector_snapshot_index = cfg["detector_snapshotindex"]

    for train_set_index in train_set_indices:
        for shuffle in shuffles:
            loader = DLCLoader(
                config=Path(cfg["project_path"]) / "config.yaml",
                shuffle=shuffle,
                trainset_index=train_set_index,
                modelprefix=modelprefix,
            )
            loader.evaluation_folder.mkdir(exist_ok=True, parents=True)

            if device is not None:
                loader.model_cfg["device"] = device
            loader.model_cfg["device"] = utils.resolve_device(loader.model_cfg)

            task = Task(loader.model_cfg["method"])
            snapshots = get_model_snapshots(
                snapshotindex,
                model_folder=loader.model_folder,
                task=task,
            )

            detector_snapshots = [None]
            if task == Task.TOP_DOWN:
                if detector_snapshot_index is not None:
                    det_snapshots = get_model_snapshots(
                        "all", loader.model_folder, Task.DETECT
                    )
                    if len(det_snapshots) == 0:
                        print(
                            "The detector_snapshot_index was set to "
                            f"{detector_snapshot_index} but no detector snapshots were "
                            f"found in {loader.model_folder}. Using ground truth "
                            "bounding boxes to compute metrics.\n"
                            "To analyze videos with a top-down model, you'll need to "
                            "train a detector!"
                        )
                    else:
                        detector_snapshots = get_model_snapshots(
                            detector_snapshot_index,
                            loader.model_folder,
                            Task.DETECT,
                        )
                else:
                    print("Using GT bounding boxes to compute evaluation metrics")

            for detector_snapshot in detector_snapshots:
                for snapshot in snapshots:
                    scorer = get_scorer_name(
                        cfg=cfg,
                        shuffle=shuffle,
                        train_fraction=loader.train_fraction,
                        snapshot_uid=get_scorer_uid(snapshot, detector_snapshot),
                        modelprefix=modelprefix,
                    )
                    evaluate_snapshot(
                        loader=loader,
                        cfg=cfg,
                        scorer=scorer,
                        snapshot=snapshot,
                        transform=transform,
                        plotting=plotting,
                        show_errors=show_errors,
                        detector_snapshot=detector_snapshot,
                    )


def image_to_dlc_df_index(image: str) -> tuple[str, ...]:
    """
    Args:
        image: the path of the image to map to a DLC index

    Returns:
        the image index to create a multi-animal DLC dataframe:
            ("labeled-data", video_name, image_name)
    """
    image_path = Path(image)
    if len(image_path.parts) >= 3 and image_path.parts[-3] == "labeled-data":
        return Path(image_path).parts[-3:]

    raise ValueError(f"Unexpected image filepath for a DLC project")


def save_evaluation_results(
    df_scores: pd.DataFrame, scores_path: Path, print_results: bool, pcutoff: float
) -> None:
    """
    Saves the evaluation results to a CSV file. Adds the evaluation results for the
    model to the combined results file, or creates it if it does not yet exist.

    Args:
        df_scores: the scores dataframe for a snapshot
        scores_path: the path where the model scores CSV should be saved
        print_results: whether to print evaluation results to the console
        pcutoff: the pcutoff used to get the evaluation results
    """
    if print_results:
        print(f"Evaluation results for {scores_path.name} (pcutoff: {pcutoff}):")
        print(df_scores.iloc[0])

    # Save scores file
    df_scores.to_csv(scores_path)

    # Update combined results
    combined_scores_path = scores_path.parent.parent / "CombinedEvaluation-results.csv"
    if combined_scores_path.exists():
        df_existing_results = pd.read_csv(
            combined_scores_path, index_col=[0, 1, 2, 3, 4]
        )
        df_scores = df_scores.combine_first(df_existing_results)

    df_scores = df_scores.sort_index()
    df_scores.to_csv(combined_scores_path)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", type=str)
    parser.add_argument("--modelprefix", type=str, default="")
    parser.add_argument("--snapshotindex", type=int, default=49)
    parser.add_argument("--plotting", type=bool, default=False)
    parser.add_argument("--show_errors", type=bool, default=True)
    args = parser.parse_args()
    evaluate_network(
        config=args.config,
        modelprefix=args.modelprefix,
        snapshotindex=args.snapshotindex,
        plotting=args.plotting,
        show_errors=args.show_errors,
    )

[n-poulsen]

Great thanks - just one question, in your output files (the .csv containing the analyzed data, or the labeled video) do you get predictions for all 27 keypoints or just the 6 that you defined in your project?

[Chap5732]

Great thanks - just one question, in your output files (the .csv containing the analyzed data, or the labeled video) do you get predictions for all 27 keypoints or just the 6 that you defined in your project?

Hi, after changing the resource code I get predictions for the 6 feature points that I defined in the analyze videos ane labeled videos output.

[n-poulsen]

Thanks @Chap5732 - I've found the issue for analyze_videos: even though the model is trained with memory replay (and predicts bodyparts for all SuperAnimal keypoints), the keypoints defined in the project are used to create the predictions DataFrame (so the numbers don't match). I'll work on fixing this, thanks for flagging it!