Add test script

open-mmlab · Aug 28, 2021 · acf84ef · acf84ef
1 parent c821ce5
commit acf84ef
Show file tree

Hide file tree

Showing 2 changed files with 245 additions and 42 deletions.
diff --git a/.dev_scripts/gen_benchmark_script.py b/.dev_scripts/gen_benchmark_script.py
@@ -30,6 +30,12 @@ def parse_args():
         default=5,
         help='CPUs per task config for slurm, '
         'should be set according to your slurm environment')
+    parser.add_argument(
+        '--gpus',
+        type=int,
+        default=8,
+        help='Totally used num of GPUs config for slurm (in testing), '
+        'should be set according to your slurm environment')
     parser.add_argument(
         '--mode', type=str, default='train', help='Train or test')
     parser.add_argument(
@@ -74,45 +80,114 @@ def parse_args():
 
 '''
 
-for i, config_path in enumerate(config_paths):
-    root_dir = osp.dirname(osp.dirname(osp.abspath(__file__)))
-    if not osp.exists(osp.join(root_dir, config_path)):
-        print(f'Invalid config path (does not exist):\n{config_path}')
-        continue
-
-    config_name = config_path.split('/')[-1][:-3]
-    match_obj = re.match(r'^.*_[0-9]+x([0-9]+)_.*$', config_name)
-    if match_obj is None:
-        print(f'Invalid config path (no GPU num in '
-              f'config name):\n{config_path}')
-        continue
-
-    gpu_num = int(match_obj.group(1))
-    work_dir_name = config_path if args.long_work_dir else config_name
-
-    script += f"echo '{config_path}' &\n"
-    if args.full_log:
-        script += f'mkdir -p $CHECKPOINT_DIR/{work_dir_name}\n'
-
-    # training commands
-    script += f'GPUS={gpu_num} GPUS_PER_NODE={args.gpus_per_node} ' \
-              f'CPUS_PER_TASK={args.cpus_per_task} ' \
-              f'./tools/slurm_train.sh $PARTITION {config_name} ' \
-              f'{config_path} \\\n'
-    script += f'$CHECKPOINT_DIR/{work_dir_name} --cfg-options ' \
-              f'checkpoint_config.max_keep_ckpts={args.max_keep_ckpts} \\\n' \
-
-    # if output full log, redirect stdout and stderr to
-    # another log file in work dir
-    if args.full_log:
-        script += f'2>&1|tee $CHECKPOINT_DIR/{work_dir_name}/FULL_LOG.txt &\n'
-    else:
-        script += '>/dev/null &\n'
-
-    if i != len(config_paths) - 1:
-        script += '\n'
-
-    print(f'Successfully generated script for {config_name}')
-
-with open(args.output_file, 'w') as fo:
-    fo.write(script)
+if args.mode == 'train':
+    for i, config_path in enumerate(config_paths):
+        root_dir = osp.dirname(osp.dirname(osp.abspath(__file__)))
+        if not osp.exists(osp.join(root_dir, config_path)):
+            print(f'Invalid config path (does not exist):\n{config_path}')
+            continue
+
+        config_name = config_path.split('/')[-1][:-3]
+        match_obj = re.match(r'^.*_[0-9]+x([0-9]+)_.*$', config_name)
+        if match_obj is None:
+            print(f'Invalid config path (no GPU num in '
+                  f'config name):\n{config_path}')
+            continue
+
+        gpu_num = int(match_obj.group(1))
+        work_dir_name = config_path if args.long_work_dir else config_name
+
+        script += f"echo '{config_path}' &\n"
+        if args.full_log:
+            script += f'mkdir -p $CHECKPOINT_DIR/{work_dir_name}\n'
+
+        # training commands
+        script += f'GPUS={gpu_num} GPUS_PER_NODE={args.gpus_per_node} ' \
+                  f'CPUS_PER_TASK={args.cpus_per_task} ' \
+                  f'./tools/slurm_train.sh $PARTITION {config_name} ' \
+                  f'{config_path} \\\n'
+        script += f'$CHECKPOINT_DIR/{work_dir_name} --cfg-options ' \
+                  f'checkpoint_config.max_keep_ckpts=' \
+                  f'{args.max_keep_ckpts} \\\n' \
+
+        # if output full log, redirect stdout and stderr to
+        # another log file in work dir
+        if args.full_log:
+            script += f'2>&1|tee $CHECKPOINT_DIR/{work_dir_name}' \
+                      f'/FULL_LOG.txt &\n'
+        else:
+            script += '>/dev/null &\n'
+
+        if i != len(config_paths) - 1:
+            script += '\n'
+
+        print(f'Successfully generated script for {config_name}')
+
+    with open(args.output_file, 'w') as fo:
+        fo.write(script)
+
+elif args.mode == 'test':
+    for i, config_path in enumerate(config_paths):
+        root_dir = osp.dirname(osp.dirname(osp.abspath(__file__)))
+        if not osp.exists(osp.join(root_dir, config_path)):
+            print(f'Invalid config path (does not exist):\n{config_path}')
+            continue
+
+        config_name = config_path.split('/')[-1][:-3]
+
+        tasks = {
+            'scannet_seg', 'scannet', 's3dis_seg', 'sunrgbd', 'kitti', 'nus',
+            'lyft', 'waymo'
+        }
+        eval_option = None
+        for task in tasks:
+            if task in config_name:
+                eval_option = task
+                break
+        if eval_option is None:
+            print(f'Invalid config path (invalid task):\n{config_path}')
+            continue
+
+        work_dir_name = config_path if args.long_work_dir else config_name
+
+        script += f"echo '{config_path}' &\n"
+        if args.full_log:
+            script += f'mkdir -p $CHECKPOINT_DIR/{work_dir_name}\n'
+
+        # training commands
+        script += f'GPUS={args.gpus} GPUS_PER_NODE={args.gpus_per_node} ' \
+                  f'CPUS_PER_TASK={args.cpus_per_task} ' \
+                  f'./tools/slurm_test.sh $PARTITION {config_name} ' \
+                  f'{config_path} \\\n'
+        script += f'$CHECKPOINT_DIR/{work_dir_name}/latest.pth ' \
+
+        if eval_option in ['scannet_seg', 's3dis_seg']:
+            script += '--eval mIoU \\\n'
+        elif eval_option in ['scannet', 'sunrgbd', 'kitti', 'nus']:
+            script += '--eval map \\\n'
+        elif eval_option in ['lyft']:
+            script += f'--format-only --eval-options jsonfile_prefix=' \
+                      f'$CHECKPOINT_DIR/{work_dir_name}/results_challenge ' \
+                      f'csv_savepath=$CHECKPOINT_DIR/{work_dir_name}/' \
+                      f'results_challenge.csv \\\n'
+        elif eval_option in ['waymo']:
+            script += f'--eval waymo --eval-options pklfile_prefix=' \
+                      f'$CHECKPOINT_DIR/{work_dir_name}/kitti_results ' \
+                      f'submission_prefix=$CHECKPOINT_DIR/{work_dir_name}/' \
+                      f'kitti_results \\\n'
+
+        # if output full log, redirect stdout and stderr to
+        # another log file in work dir
+        if args.full_log:
+            script += f'2>&1|tee $CHECKPOINT_DIR/{work_dir_name}' \
+                      f'/FULL_LOG.txt &\n'
+        else:
+            script += '>/dev/null &\n'
+
+        if i != len(config_paths) - 1:
+            script += '\n'
+
+        print(f'Successfully generated script for {config_name}')
+
+    with open(args.output_file, 'w') as fo:
+        fo.write(script)
diff --git a/.dev_scripts/test_benchmark.sh b/.dev_scripts/test_benchmark.sh
@@ -0,0 +1,128 @@
+PARTITION=$1
+CHECKPOINT_DIR=$2
+
+echo 'configs/3dssd/3dssd_4x4_kitti-3d-car.py' &
+mkdir -p $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION 3dssd_4x4_kitti-3d-car configs/3dssd/3dssd_4x4_kitti-3d-car.py \
+$CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/FULL_LOG.txt &
+
+echo 'configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py' &
+mkdir -p $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py \
+$CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/FULL_LOG.txt &
+
+echo 'configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py \
+$CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/FULL_LOG.txt &
+
+echo 'configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py' &
+mkdir -p $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py \
+$CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/FULL_LOG.txt &
+
+echo 'configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py \
+$CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt &
+
+echo 'configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py' &
+mkdir -p $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py \
+$CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/FULL_LOG.txt &
+
+echo 'configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py' &
+mkdir -p $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION groupfree3d_8x4_scannet-3d-18class-L6-O256 configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py \
+$CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/FULL_LOG.txt &
+
+echo 'configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION h3dnet_3x8_scannet-3d-18class configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py \
+$CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/FULL_LOG.txt &
+
+echo 'configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py \
+$CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/FULL_LOG.txt &
+
+echo 'configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvotenet_stage2_16x8_sunrgbd-3d-10class configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py \
+$CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/FULL_LOG.txt &
+
+echo 'configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py' &
+mkdir -p $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvoxelnet_4x8_kitti-3d-car configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py \
+$CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/FULL_LOG.txt &
+
+echo 'configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py \
+$CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/FULL_LOG.txt &
+
+echo 'configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py \
+$CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/FULL_LOG.txt &
+
+echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py \
+$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/latest.pth --eval mIoU \
+2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/FULL_LOG.txt &
+
+echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py \
+$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/FULL_LOG.txt &
+
+echo 'configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py' &
+mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py \
+$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/latest.pth --format-only --eval-options jsonfile_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/results_challenge csv_savepath=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/results_challenge.csv \
+2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/FULL_LOG.txt &
+
+echo 'configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py \
+$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/latest.pth --eval waymo --eval-options pklfile_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/kitti_results submission_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/kitti_results \
+2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/FULL_LOG.txt &
+
+echo 'configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py' &
+mkdir -p $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py \
+$CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/FULL_LOG.txt &
+
+echo 'configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_second_secfpn_6x8_80e_kitti-3d-3class configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py \
+$CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt &
+
+echo 'configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py' &
+mkdir -p $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py \
+$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/latest.pth --format-only --eval-options jsonfile_prefix=$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/results_challenge csv_savepath=$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/results_challenge.csv \
+2>&1|tee $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/FULL_LOG.txt &
+
+echo 'configs/votenet/votenet_8x8_scannet-3d-18class.py' &
+mkdir -p $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py
+GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION votenet_8x8_scannet-3d-18class configs/votenet/votenet_8x8_scannet-3d-18class.py \
+$CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/latest.pth --eval map \
+2>&1|tee $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/FULL_LOG.txt &