diff --git a/mmpretrain/datasets/__init__.py b/mmpretrain/datasets/__init__.py
index 2c81e35a68f..5f3b5bae6b7 100644
--- a/mmpretrain/datasets/__init__.py
+++ b/mmpretrain/datasets/__init__.py
@@ -24,14 +24,13 @@
 from .sun397 import SUN397
 from .transforms import *  # noqa: F401,F403
 from .voc import VOC
-from .vsr import VSR
 
 __all__ = [
     'BaseDataset', 'CIFAR10', 'CIFAR100', 'CUB', 'Caltech101', 'CustomDataset',
     'DTD', 'FGVCAircraft', 'FashionMNIST', 'Flowers102', 'Food101', 'ImageNet',
     'ImageNet21k', 'InShop', 'KFoldDataset', 'MNIST', 'MultiLabelDataset',
     'MultiTaskDataset', 'NLVR2', 'OxfordIIITPet', 'Places205', 'SUN397',
-    'StanfordCars', 'VOC', 'VSR', 'build_dataset'
+    'StanfordCars', 'VOC', 'build_dataset'
 ]
 
 if WITH_MULTIMODAL:
@@ -45,10 +44,10 @@
     from .scienceqa import ScienceQA
     from .textvqa import TextVQA
     from .visual_genome import VisualGenomeQA
+    from .vsr import VSR
 
     __all__.extend([
         'COCOCaption', 'COCORetrieval', 'COCOVQA', 'FlamingoEvalCOCOCaption',
         'FlamingoEvalCOCOVQA', 'RefCOCO', 'VisualGenomeQA', 'ScienceQA',
-        'NoCaps'
-        'GQA', 'TextVQA'
+        'NoCaps', 'GQA', 'TextVQA', 'VSR'
     ])
diff --git a/mmpretrain/datasets/vsr.py b/mmpretrain/datasets/vsr.py
index 5f67765975b..7b109592bd0 100644
--- a/mmpretrain/datasets/vsr.py
+++ b/mmpretrain/datasets/vsr.py
@@ -1,5 +1,4 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-import os.path as osp
 from typing import List
 
 import mmengine
@@ -41,17 +40,15 @@ def load_data_list(self) -> List[dict]:
         for ann in annotations:
             # ann example
             # {
-            #     'question': "Is it overcast?",
-            #     'answer': 'no,
-            #     'image_id': n161313.jpg,
-            #     'question_id': 262148000,
-            #     ....
+            #     "image": "train2017/000000372029.jpg",
+            #     "question": "The dog is on the surfboard.",
+            #     "answer": true
             # }
             data_info = dict()
-            data_info['img_path'] = osp.join(self.data_prefix['img_path'],
-                                             ann['image'])
+            data_info['img_path'] = mmengine.join_path(
+                self.data_prefix['img_path'], ann['image'])
             data_info['question'] = ann['question']
-            data_info['gt_answer'] = ann['answer']
+            data_info['gt_answer'] = 'yes' if ann['answer'] else 'no'
 
             data_list.append(data_info)