v1.1 code for RODNet J-STSP version

README.md

@@ -1,53 +1,78 @@
-# RODNet: Radar Object Detection using Cross-Modal Supervision
+# RODNet: Radar Object Detection Network
 
-This is the official implementation of our RODNet paper at WACV 2021. 
+This is the official implementation of our RODNet papers 
+at [WACV 2021](https://openaccess.thecvf.com/content/WACV2021/html/Wang_RODNet_Radar_Object_Detection_Using_Cross-Modal_Supervision_WACV_2021_paper.html) 
+and [IEEE J-STSP 2021](https://ieeexplore.ieee.org/abstract/document/9353210). 
 
-[[Paper]](https://openaccess.thecvf.com/content/WACV2021/html/Wang_RODNet_Radar_Object_Detection_Using_Cross-Modal_Supervision_WACV_2021_paper.html)
+[[Arxiv]](https://arxiv.org/abs/2102.05150)
 [[Dataset]](https://www.cruwdataset.org)
+[[ROD2021 Challenge]](https://codalab.lisn.upsaclay.fr/competitions/1063)
+[[Presentation]](https://youtu.be/UZbxI4o2-7g)
+[[Demo]](https://youtu.be/09HaDySa29I)
 
 ![RODNet Overview](./assets/images/overview.jpg?raw=true)
 
-Please cite our WACV 2021 paper if this repository is helpful for your research:
+Please cite our paper if this repository is helpful for your research:
 ```
 @inproceedings{wang2021rodnet,
-    author={Wang, Yizhou and Jiang, Zhongyu and Gao, Xiangyu and Hwang, Jenq-Neng and Xing, Guanbin and Liu, Hui},
-    title={RODNet: Radar Object Detection Using Cross-Modal Supervision},
-    booktitle={Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)},
-    month={January},
-    year={2021},
-    pages={504-513}
+  author={Wang, Yizhou and Jiang, Zhongyu and Gao, Xiangyu and Hwang, Jenq-Neng and Xing, Guanbin and Liu, Hui},
+  title={RODNet: Radar Object Detection Using Cross-Modal Supervision},
+  booktitle={Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)},
+  month={January},
+  year={2021},
+  pages={504-513}
 }
 ```
+```
+@article{wang2021rodnet,
+  title={RODNet: A Real-Time Radar Object Detection Network Cross-Supervised by Camera-Radar Fused Object 3D Localization},
+  author={Wang, Yizhou and Jiang, Zhongyu and Li, Yudong and Hwang, Jenq-Neng and Xing, Guanbin and Liu, Hui},
+  journal={IEEE Journal of Selected Topics in Signal Processing},
+  volume={15},
+  number={4},
+  pages={954--967},
+  year={2021},
+  publisher={IEEE}
+}
+```
+
 ## Installation
 
 Create a conda environment for RODNet. Tested under Python 3.6, 3.7, 3.8.
-```
+```commandline
 conda create -n rodnet python=3.* -y
 conda activate rodnet
 ```
 
 Install pytorch.
-```
-conda install pytorch torchvision cudatoolkit=10.1 -c pytorch
+**Note:** If you are using Temporal Deformable Convolution (TDC), we only tested under `pytorch<=1.4` and `CUDA=10.1`. 
+Without TDC, you should be able to choose the latest versions. 
+If you met some issues with environment, feel free to raise an issue.
+```commandline
+conda install pytorch=1.4 torchvision cudatoolkit=10.1 -c pytorch  # if using TDC
+# OR
+conda install pytorch torchvision cudatoolkit=10.1 -c pytorch  # if not using TDC
 ```
 
 Install `cruw-devkit` package. 
 Please refer to [`cruw-devit`](https://github.com/yizhou-wang/cruw-devkit) repository for detailed instructions.
-```
+```commandline
 git clone https://github.com/yizhou-wang/cruw-devkit.git
 cd cruw-devkit
 pip install -e .
 cd ..
 ```
 
 Setup RODNet package.
-```
+```commandline
 pip install -e .
 ```
+**Note:** If you are not using TDC, you can rename script `setup_wo_tdc.py` as `setup.py`, and run the above command. 
+This should allow you to use the latest cuda and pytorch version. 
 
 ## Prepare data for RODNet
 
-```
+```commandline
 python tools/prepare_dataset/prepare_data.py \
         --config configs/<CONFIG_FILE> \
         --data_root <DATASET_ROOT> \
@@ -57,15 +82,15 @@ python tools/prepare_dataset/prepare_data.py \
 
 ## Train models
 
-```
+```commandline
 python tools/train.py --config configs/<CONFIG_FILE> \
         --data_dir data/<DATA_FOLDER_NAME> \
         --log_dir checkpoints/
 ```
 
 ## Inference
 
-```
+```commandline
 python tools/test.py --config configs/<CONFIG_FILE> \
         --data_dir data/<DATA_FOLDER_NAME> \
         --checkpoint <CHECKPOINT_PATH> \

configs/config_rodnet_v2_cdc_win16.py

@@ -0,0 +1,77 @@
+dataset_cfg = dict(
+    dataset_name='ROD2021',
+    base_root="/mnt/disk2/CRUW/ROD2021",
+    data_root="/mnt/disk2/CRUW/ROD2021/sequences",
+    anno_root="/mnt/disk2/CRUW/ROD2021/annotations",
+    anno_ext='.txt',
+    train=dict(
+        subdir='train',
+        # seqs=[],  # can choose from the subdir folder
+    ),
+    valid=dict(
+        subdir='valid',
+        seqs=[],
+    ),
+    test=dict(
+        subdir='test',
+        # seqs=[],  # can choose from the subdir folder
+    ),
+    demo=dict(
+        subdir='demo',
+        seqs=[],
+    ),
+)
+
+model_cfg = dict(
+    type='CDCv2',
+    name='rodnet-cdc-v2-win16-wobg',
+    max_dets=20,
+    peak_thres=0.3,
+    ols_thres=0.3,
+    mnet_cfg=(4, 32),
+    dcn=True,
+)
+
+confmap_cfg = dict(
+    confmap_sigmas={
+        'pedestrian': 15,
+        'cyclist': 20,
+        'car': 30,
+        # 'van': 40,
+        # 'truck': 50,
+    },
+    confmap_sigmas_interval={
+        'pedestrian': [5, 15],
+        'cyclist': [8, 20],
+        'car': [10, 30],
+        # 'van': [15, 40],
+        # 'truck': [20, 50],
+    },
+    confmap_length={
+        'pedestrian': 1,
+        'cyclist': 2,
+        'car': 3,
+        # 'van': 4,
+        # 'truck': 5,
+    }
+)
+
+train_cfg = dict(
+    n_epoch=100,
+    batch_size=4,
+    lr=0.00001,
+    lr_step=5,  # lr will decrease 10 times after lr_step epoches
+    win_size=16,
+    train_step=1,
+    train_stride=4,
+    log_step=100,
+    save_step=10000,
+)
+test_cfg = dict(
+    test_step=1,
+    test_stride=8,
+    rr_min=1.0,  # min radar range
+    rr_max=20.0,  # max radar range
+    ra_min=-60.0,  # min radar angle
+    ra_max=60.0,  # max radar angle
+)

configs/config_rodnet_v2_hg1_win16.py

@@ -0,0 +1,78 @@
+dataset_cfg = dict(
+    dataset_name='ROD2021',
+    base_root="/mnt/disk2/CRUW/ROD2021",
+    data_root="/mnt/disk2/CRUW/ROD2021/sequences",
+    anno_root="/mnt/disk2/CRUW/ROD2021/annotations",
+    anno_ext='.txt',
+    train=dict(
+        subdir='train',
+        # seqs=[],  # can choose from the subdir folder
+    ),
+    valid=dict(
+        subdir='valid',
+        seqs=[],
+    ),
+    test=dict(
+        subdir='test',
+        # seqs=[],  # can choose from the subdir folder
+    ),
+    demo=dict(
+        subdir='demo',
+        seqs=[],
+    ),
+)
+
+model_cfg = dict(
+    type='HGv2',
+    name='rodnet-hg1-v2-win16-wobg',
+    max_dets=20,
+    peak_thres=0.4,
+    ols_thres=0.3,
+    stacked_num=1,
+    mnet_cfg=(4, 32),
+    dcn=True,
+)
+
+confmap_cfg = dict(
+    confmap_sigmas={
+        'pedestrian': 15,
+        'cyclist': 20,
+        'car': 30,
+        # 'van': 40,
+        # 'truck': 50,
+    },
+    confmap_sigmas_interval={
+        'pedestrian': [5, 15],
+        'cyclist': [8, 20],
+        'car': [10, 30],
+        # 'van': [15, 40],
+        # 'truck': [20, 50],
+    },
+    confmap_length={
+        'pedestrian': 1,
+        'cyclist': 2,
+        'car': 3,
+        # 'van': 4,
+        # 'truck': 5,
+    }
+)
+
+train_cfg = dict(
+    n_epoch=50,
+    batch_size=4,
+    lr=0.00001,
+    lr_step=5,  # lr will decrease 10 times after lr_step epoches
+    win_size=16,
+    train_step=1,
+    train_stride=4,
+    log_step=100,
+    save_step=10000,
+)
+test_cfg = dict(
+    test_step=1,
+    test_stride=8,
+    rr_min=1.0,  # min radar range
+    rr_max=20.0,  # max radar range
+    ra_min=-60.0,  # min radar angle
+    ra_max=60.0,  # max radar angle
+)

configs/config_rodnet_v2_hg1wi_win16.py

@@ -0,0 +1,78 @@
+dataset_cfg = dict(
+    dataset_name='ROD2021',
+    base_root="/mnt/disk1/CRUW/ROD2021",
+    data_root="/mnt/disk1/CRUW/ROD2021/sequences",
+    anno_root="/mnt/disk1/CRUW/ROD2021/annotations",
+    anno_ext='.txt',
+    train=dict(
+        subdir='train',
+        # seqs=[],  # can choose from the subdir folder
+    ),
+    valid=dict(
+        subdir='valid',
+        # seqs=[],
+    ),
+    test=dict(
+        subdir='test',
+        # seqs=[],  # can choose from the subdir folder
+    ),
+    demo=dict(
+        subdir='demo',
+        seqs=[],
+    ),
+)
+
+model_cfg = dict(
+    type='HGwIv2',
+    name='rodnet-hg1wi-v2-win16-wobg',
+    max_dets=20,
+    peak_thres=0.3,
+    ols_thres=0.3,
+    stacked_num=1,
+    mnet_cfg=(4, 32),
+    dcn=True,
+)
+
+confmap_cfg = dict(
+    confmap_sigmas={
+        'pedestrian': 15,
+        'cyclist': 20,
+        'car': 30,
+        # 'van': 40,
+        # 'truck': 50,
+    },
+    confmap_sigmas_interval={
+        'pedestrian': [5, 15],
+        'cyclist': [8, 20],
+        'car': [10, 30],
+        # 'van': [15, 40],
+        # 'truck': [20, 50],
+    },
+    confmap_length={
+        'pedestrian': 1,
+        'cyclist': 2,
+        'car': 3,
+        # 'van': 4,
+        # 'truck': 5,
+    }
+)
+
+train_cfg = dict(
+    n_epoch=100,
+    batch_size=2,
+    lr=0.00001,
+    lr_step=5,  # lr will decrease 10 times after lr_step epoches
+    win_size=16,
+    train_step=1,
+    train_stride=4,
+    log_step=100,
+    save_step=10000,
+)
+test_cfg = dict(
+    test_step=1,
+    test_stride=8,
+    rr_min=1.0,  # min radar range
+    rr_max=20.0,  # max radar range
+    ra_min=-60.0,  # min radar angle
+    ra_max=60.0,  # max radar angle
+)

rodnet/datasets/CRDataLoader.py

@@ -5,14 +5,14 @@
 from multiprocessing import Array, Process
 
 
-class CRDataLoader():
+class CRDataLoader:
     def __init__(self, dataset, shuffle=False, num_parallel_batch=2, noise_channel=False):
         # parameters settings
         self.dataset = dataset
         self.config_dict = self.dataset.config_dict
-        self.n_class = self.config_dict['class_cfg']['n_class']
+        self.n_class = dataset.dataset.object_cfg.n_class
         self.batch_size = self.config_dict['train_cfg']['batch_size']
-        self.radar_configs = self.config_dict['dataset_cfg']['radar_cfg']
+        self.radar_configs = self.dataset.dataset.sensor_cfg.radar_cfg
         self.model_configs = self.config_dict['model_cfg']
         self.ramap_rsize = self.radar_configs['ramap_rsize']
         self.ramap_asize = self.radar_configs['ramap_asize']