'CIFAR10 is not in the dataset registry #699
Unanswered
mrFocusXin
asked this question in
How-to
Replies: 0 comments
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
-
I want to use CIFAR10 in ODC, and my config file as follow:
base = [
'../base/models/odc.py',
# '../base/datasets/imagenet_odc.py',
'../base/schedules/sgd_steplr-200e_in1k.py',
'../base/default_runtime.py',
]
##Changed Start
dataset settings
dataset_type = 'CIFAR10'
data_root = '/home/wangxin/cifar/'
file_client_args = dict(backend='disk')
train_pipeline = [
dict(type='LoadImageFromFile', file_client_args=file_client_args),
dict(type='RandomResizedCrop', size=224, backend='pillow'),
dict(type='RandomFlip', prob=0.5),
dict(type='RandomRotation', degrees=2),
dict(
type='ColorJitter',
brightness=0.4,
contrast=0.4,
saturation=1.0,
hue=0.5),
dict(
type='RandomGrayscale',
prob=0.2,
keep_channels=True,
channel_weights=(0.114, 0.587, 0.2989)),
dict(
type='PackSelfSupInputs',
algorithm_keys=['sample_idx'],
meta_keys=['img_path'])
]
extract_pipeline = [
dict(type='LoadImageFromFile', file_client_args=file_client_args),
dict(type='mmcls.ResizeEdge', scale=256, edge='short', backend='pillow'),
dict(type='CenterCrop', crop_size=224),
dict(type='PackSelfSupInputs', meta_keys=['img_path'])
]
train_dataloader = dict(
batch_size=64,
num_workers=4,
persistent_workers=True,
sampler=dict(type='DeepClusterSampler', shuffle=True, replace=True),
collate_fn=dict(type='default_collate'),
dataset=dict(
type=dataset_type,
data_root=data_root,
ann_file='meta/train.txt',
data_prefix=dict(img_path='train/'),
pipeline=train_pipeline))
num_classes = 10000
custom_hooks = [
dict(
type='DeepClusterHook',
extract_dataloader=dict(
batch_size=128,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=False, round_up=True),
collate_fn=dict(type='default_collate'),
dataset=dict(
type=dataset_type,
data_root=data_root,
ann_file='meta/train.txt',
data_prefix=dict(img_path='train/'),
pipeline=extract_pipeline)),
clustering=dict(type='Kmeans', k=num_classes, pca_dim=-1), # no pca
unif_sampling=False,
reweight=True,
reweight_pow=0.5,
init_memory=True,
initial=True, # call initially
interval=9999999999), # initial only
dict(
type='ODCHook',
centroids_update_interval=10, # iter
deal_with_small_clusters_interval=1,
evaluate_interval=50,
reweight=True,
reweight_pow=0.5)
]
##Changed End
model settings
model = dict(
)
optimizer
optimizer = dict(type='SGD', lr=0.06, weight_decay=1e-5, momentum=0.9)
optim_wrapper = dict(
type='OptimWrapper',
optimizer=optimizer,
paramwise_cfg=dict(custom_keys={'head': dict(momentum=0.)}))
learning rate scheduler
param_scheduler = [
dict(type='MultiStepLR', by_epoch=True, milestones=[400], gamma=0.4)
]
runtime settings
train_cfg = dict(max_epochs=440)
default_hooks = dict(
checkpoint=dict(type='CheckpointHook', interval=10, max_keep_ckpts=3))
Then there has Error as follow :
02/21 02:17:01 - mmengine - INFO -
System environment:
sys.platform: linux
Python: 3.8.15 (default, Nov 11 2022, 14:08:18) [GCC 11.2.0]
CUDA available: False
numpy_random_seed: 889434048
GCC: gcc (Ubuntu 11.3.0-1ubuntu1~22.04) 11.3.0
PyTorch: 1.13.1
PyTorch compiling details: PyTorch built with:
GCC 9.3
C++ Version: 201402
Intel(R) oneAPI Math Kernel Library Version 2023.0-Product Build 20221128 for Intel(R) 64 architecture applications
Intel(R) MKL-DNN v2.6.0 (Git Hash 52b5f107dd9cf10910aaa19cb47f3abf9b349815)
OpenMP 201511 (a.k.a. OpenMP 4.5)
LAPACK is enabled (usually provided by MKL)
NNPACK is enabled
CPU capability usage: AVX2
Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -fabi-version=11 -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -fopenmp -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOCUPTI -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -DEDGE_PROFILER_USE_KINETO -O2 -fPIC -Wno-narrowing -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wunused-local-typedefs -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Werror=cast-function-type -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_VERSION=1.13.1, USE_CUDA=0, USE_CUDNN=OFF, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=OFF, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF,
TorchVision: 0.14.1
OpenCV: 4.7.0
MMEngine: 0.5.0
Runtime environment:
cudnn_benchmark: False
mp_cfg: {'mp_start_method': 'fork', 'opencv_num_threads': 0}
dist_cfg: {'backend': 'nccl'}
seed: None
Distributed launcher: none
Distributed training: False
GPU number: 1
02/21 02:17:01 - mmengine - INFO - Config:
model = dict(
type='ODC',
data_preprocessor=dict(
mean=(123.675, 116.28, 103.53),
std=(58.395, 57.12, 57.375),
bgr_to_rgb=True),
backbone=dict(
type='ResNet',
depth=50,
in_channels=3,
out_indices=[4],
norm_cfg=dict(type='SyncBN')),
neck=dict(
type='ODCNeck',
in_channels=2048,
hid_channels=512,
out_channels=256,
with_avg_pool=True),
head=dict(
type='ClsHead',
loss=dict(type='mmcls.CrossEntropyLoss'),
with_avg_pool=False,
in_channels=256,
num_classes=10000),
memory_bank=dict(
type='ODCMemory',
length=1281167,
feat_dim=256,
momentum=0.5,
num_classes=10000,
min_cluster=20,
debug=False))
optimizer = dict(type='SGD', lr=0.06, weight_decay=1e-05, momentum=0.9)
optim_wrapper = dict(
type='OptimWrapper',
optimizer=dict(type='SGD', lr=0.06, weight_decay=1e-05, momentum=0.9),
paramwise_cfg=dict(custom_keys=dict(head=dict(momentum=0.0))))
param_scheduler = [
dict(type='MultiStepLR', by_epoch=True, milestones=[400], gamma=0.4)
]
train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=440)
default_scope = 'mmselfsup'
default_hooks = dict(
runtime_info=dict(type='RuntimeInfoHook'),
timer=dict(type='IterTimerHook'),
logger=dict(type='LoggerHook', interval=50),
param_scheduler=dict(type='ParamSchedulerHook'),
checkpoint=dict(type='CheckpointHook', interval=10, max_keep_ckpts=3),
sampler_seed=dict(type='DistSamplerSeedHook'))
env_cfg = dict(
cudnn_benchmark=False,
mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
dist_cfg=dict(backend='nccl'))
log_processor = dict(
window_size=10,
custom_cfg=[dict(data_src='', method='mean', window_size='global')])
vis_backends = [dict(type='LocalVisBackend')]
visualizer = dict(
type='SelfSupVisualizer',
vis_backends=[dict(type='LocalVisBackend')],
name='visualizer')
log_level = 'INFO'
load_from = None
resume = False
dataset_type = 'CIFAR10'
data_root = '/home/wangxin/cifar/'
file_client_args = dict(backend='disk')
train_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='RandomResizedCrop', size=224, backend='pillow'),
dict(type='RandomFlip', prob=0.5),
dict(type='RandomRotation', degrees=2),
dict(
type='ColorJitter',
brightness=0.4,
contrast=0.4,
saturation=1.0,
hue=0.5),
dict(
type='RandomGrayscale',
prob=0.2,
keep_channels=True,
channel_weights=(0.114, 0.587, 0.2989)),
dict(
type='PackSelfSupInputs',
algorithm_keys=['sample_idx'],
meta_keys=['img_path'])
]
extract_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='mmcls.ResizeEdge', scale=256, edge='short', backend='pillow'),
dict(type='CenterCrop', crop_size=224),
dict(type='PackSelfSupInputs', meta_keys=['img_path'])
]
train_dataloader = dict(
batch_size=64,
num_workers=4,
persistent_workers=True,
sampler=dict(type='DeepClusterSampler', shuffle=True, replace=True),
collate_fn=dict(type='default_collate'),
dataset=dict(
type='CIFAR10',
data_root='/home/wangxin/cifar/',
ann_file='meta/train.txt',
data_prefix=dict(img_path='train/'),
pipeline=[
dict(
type='LoadImageFromFile',
file_client_args=dict(backend='disk')),
dict(type='RandomResizedCrop', size=224, backend='pillow'),
dict(type='RandomFlip', prob=0.5),
dict(type='RandomRotation', degrees=2),
dict(
type='ColorJitter',
brightness=0.4,
contrast=0.4,
saturation=1.0,
hue=0.5),
dict(
type='RandomGrayscale',
prob=0.2,
keep_channels=True,
channel_weights=(0.114, 0.587, 0.2989)),
dict(
type='PackSelfSupInputs',
algorithm_keys=['sample_idx'],
meta_keys=['img_path'])
]))
num_classes = 10000
custom_hooks = [
dict(
type='DeepClusterHook',
extract_dataloader=dict(
batch_size=128,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=False, round_up=True),
collate_fn=dict(type='default_collate'),
dataset=dict(
type='CIFAR10',
data_root='/home/wangxin/cifar/',
ann_file='meta/train.txt',
data_prefix=dict(img_path='train/'),
pipeline=[
dict(
type='LoadImageFromFile',
file_client_args=dict(backend='disk')),
dict(
type='mmcls.ResizeEdge',
scale=256,
edge='short',
backend='pillow'),
dict(type='CenterCrop', crop_size=224),
dict(type='PackSelfSupInputs', meta_keys=['img_path'])
])),
clustering=dict(type='Kmeans', k=10000, pca_dim=-1),
unif_sampling=False,
reweight=True,
reweight_pow=0.5,
init_memory=True,
initial=True,
interval=9999999999),
dict(
type='ODCHook',
centroids_update_interval=10,
deal_with_small_clusters_interval=1,
evaluate_interval=50,
reweight=True,
reweight_pow=0.5)
]
launcher = 'none'
work_dir = './work_dirs/selfsup/odc_resnet50_8xb64-steplr-440e_cifar'
02/21 02:17:01 - mmengine - WARNING - The "visualizer" registry in mmselfsup did not set import location. Fallback to call
mmselfsup.utils.register_all_modulesinstead.02/21 02:17:01 - mmengine - WARNING - The "vis_backend" registry in mmselfsup did not set import location. Fallback to call
mmselfsup.utils.register_all_modulesinstead.02/21 02:17:02 - mmengine - WARNING - The "model" registry in mmselfsup did not set import location. Fallback to call
mmselfsup.utils.register_all_modulesinstead.02/21 02:17:02 - mmengine - WARNING - The "model" registry in mmcls did not set import location. Fallback to call
mmcls.utils.register_all_modulesinstead.02/21 02:17:02 - mmengine - INFO - Distributed training is not used, all SyncBatchNorm (SyncBN) layers in the model will be automatically reverted to BatchNormXd layers if they are used.
02/21 02:17:02 - mmengine - WARNING - The "hook" registry in mmselfsup did not set import location. Fallback to call
mmselfsup.utils.register_all_modulesinstead.02/21 02:17:02 - mmengine - WARNING - The "dataset" registry in mmselfsup did not set import location. Fallback to call
mmselfsup.utils.register_all_modulesinstead.Traceback (most recent call last):
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
File "/home/wangxin/mmselfsup/mmselfsup/engine/hooks/deepcluster_hook.py", line 47, in init
self.extractor = Extractor(
File "/home/wangxin/mmselfsup/mmselfsup/models/utils/extractor.py", line 52, in init
self.data_loader = Runner.build_dataloader(
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/runner/runner.py", line 1333, in build_dataloader
dataset = DATASETS.build(dataset_cfg)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/registry/registry.py", line 521, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/registry/build_functions.py", line 100, in build_from_cfg
raise KeyError(
KeyError: 'CIFAR10 is not in the dataset registry. Please check whether the value of
CIFAR10is correct or it was registered as expected. More details can be found at https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html#import-the-custom-module'During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "tools/train.py", line 99, in
main()
File "tools/train.py", line 92, in main
runner = Runner.from_cfg(cfg)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/runner/runner.py", line 431, in from_cfg
runner = cls(
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/runner/runner.py", line 411, in init
self.register_hooks(default_hooks, custom_hooks)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/runner/runner.py", line 1904, in register_hooks
self.register_custom_hooks(custom_hooks)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/runner/runner.py", line 1885, in register_custom_hooks
self.register_hook(hook)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/runner/runner.py", line 1786, in register_hook
hook_obj = HOOKS.build(hook)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/registry/registry.py", line 521, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/home/wangxin/anaconda3/envs/openmmlab/lib/python3.8/site-packages/mmengine/registry/build_functions.py", line 135, in build_from_cfg
raise type(e)(
KeyError: "class
DeepClusterHookin mmselfsup/engine/hooks/deepcluster_hook.py: 'CIFAR10 is not in the dataset registry. Please check whether the value ofCIFAR10is correct or it was registered as expected. More details can be found at https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html#import-the-custom-module'"It's seems need to registry 'CIFAR10'? Does mmselfsup have its own CIFAR10? How to use? Thanks for help!
Beta Was this translation helpful? Give feedback.
All reactions