-
Notifications
You must be signed in to change notification settings - Fork 0
/
multi_node_eval.sh
23 lines (23 loc) · 11.4 KB
/
multi_node_eval.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# submit on local first
MODEL_BASE="lmsys/vicuna-7b-v1.5" # only works for lora-model
MODEL_PATH="./checkpoints/llava-v1.5-7b-ms-ret-0.2-trm-fp16"
PRETRAIN_MM_MLP_ADAPTER="None"
LOG_PATH="./oven_eval_logs_7b_in_all"
CONSTRAINED_DECODE_TYPE="raw_ret"
./create_screens.sh eval0 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=0 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_0.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_0.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_0.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_0.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_0.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 0 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
./create_screens.sh eval1 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=1 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_1.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_1.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_1.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_1.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_1.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 1 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
./create_screens.sh eval2 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=2 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_2.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_2.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_2.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_2.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_2.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 2 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
./create_screens.sh eval3 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=3 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_3.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_3.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_3.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_3.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_3.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 3 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
./create_screens.sh eval4 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=4 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_4.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_4.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_4.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_4.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_4.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 4 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
./create_screens.sh eval5 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=5 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_5.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_5.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_5.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_5.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_5.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 5 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
./create_screens.sh eval6 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=6 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_6.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_6.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_6.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_6.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_6.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 6 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
./create_screens.sh eval7 "cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=7 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_7.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_7.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_7.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_7.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_7.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 7 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}"
# then goes to node1
pdsh -w node-1 "cp /vc_data/users/v-zilinxiao/bk.bashrc ~/.bashrc; cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval0 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=0 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_8.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_8.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_8.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_8.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_8.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 8 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"
pdsh -w node-1 "cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval1 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=1 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_9.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_9.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_9.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_9.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_9.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 9 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"
pdsh -w node-1 "cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval2 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=2 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_10.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_10.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_10.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_10.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_10.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 10 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"
pdsh -w node-1 "cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval3 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=3 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_11.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_11.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_11.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_11.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_11.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 11 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"
pdsh -w node-1 "cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval4 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=4 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_12.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_12.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_12.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_12.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_12.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 12 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"
pdsh -w node-1 "cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval5 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=5 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_13.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_13.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_13.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_13.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_13.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 13 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"
pdsh -w node-1 "cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval6 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=6 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_14.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_14.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_14.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_14.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_14.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 14 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"
pdsh -w node-1 "cd /vc_data/users/v-zilinxiao/code/v_entity; ./create_screens.sh eval7 'cd /vc_data/users/v-zilinxiao/code/v_entity; CUDA_VISIBLE_DEVICES=7 python eval_oven_entity_ret.py --val_jsonl_path ./ms_jsonl_dataset/oven_entity_test_labeled_ms_t16_15.jsonl ./ms_jsonl_dataset/oven_entity_val_ms_t16_15.jsonl ./ms_jsonl_dataset/oven_human_labeled_ms_t16_15.jsonl ./ms_jsonl_dataset/oven_query_test_labeled_ms_t16_15.jsonl ./ms_jsonl_dataset/oven_query_val_ms_t16_15.jsonl --img_shards_path /vc_data/users/v-zilinxiao/dataset/entity_oven/ --fake_dist 15 --model_base ${MODEL_BASE} --model_path ${MODEL_PATH} --log_path ${LOG_PATH} --pretrain_mm_mlp_adapter ${PRETRAIN_MM_MLP_ADAPTER} --constrained_decode_type ${CONSTRAINED_DECODE_TYPE}'"