Merge branch 'master' of github.com:FlagOpen/FlagEmbedding

lucataco · Feb 7, 2024 · e23ff5e · e23ff5e
2 parents a7e7ece + ab86bed
commit e23ff5e
Showing 1 changed file with 50 additions and 19 deletions.
diff --git a/Long_LLM/activation_beacon/docs/evaluation.md b/Long_LLM/activation_beacon/docs/evaluation.md
@@ -36,29 +36,31 @@ data_root="/data/activation-beacon"
 
 # NOTE: in the first run, the tokenization could be super slow (often consumes half an hour). However the tokenized corpus will be saved and reused. Be patient.
 
-############## Llama-2 ##############
+################ Llama-2 ################
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 32768 --use_flash_attention_2
 
-############## PI ##############
+################ PI ################
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 32768 --use_flash_attention_2 --rope_method linear --rope_factor 8
 
-############## NTK ##############
+################ NTK ################
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 32768 --use_flash_attention_2 --rope_method dynamic --rope_factor 2
 
-############## LongLlama ##############
+################ LongLlama ################
 # OOM given 32K
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 32768 --model_name_or_path syzymon/long_llama_code_7b_instruct
 # evaluate 16K instead
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 16384 --model_name_or_path syzymon/long_llama_code_7b_instruct
 
-############## LongChat ##############
+################ LongChat ################
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 32768 --model_name_or_path lmsys/longchat-7b-v1.5-32k --use_flash_attention_2
 
-############## Activation Beacon ##############
+################ Activation Beacon ################
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 32768 --model_name_or_path namespace-Pt/activation-beacon-llama2-7b-chat
+
 # evaluating with 400K context (increase stride to 100K so sliding window evaluation is faster)
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 400000 --stride 100000 --model_name_or_path namespace-Pt/activation-beacon-llama2-7b-chat
-# evaluating with 1M (increase stride to 100K so sliding window evaluation is faster)
+
+# evaluating with 1M context (increase stride to 100K so sliding window evaluation is faster)
 torchrun --nproc_per_node 8 -m main.eval_lm --data_root $data_root --max_length 1000000 --stride 100000 --model_name_or_path namespace-Pt/activation-beacon-llama2-7b-chat --rope_method dynamic --rope_factor 2
 ```
 
@@ -77,22 +79,22 @@ The results can be found at `data/results/lm/pg19.log`.
 ```bash
 data_root="/data/activation-beacon"
 
-############## Llama-2 ##############
+################ Llama-2 ################
 torchrun --nproc_per_node 8 -m main.eval_longbench --data_root $data_root --max_length 3500 --use_flash_attention_2
 
-############## PI ##############
+################ PI ################
 torchrun --nproc_per_node 8 -m main.eval_longbench --data_root $data_root --max_length 15500 --use_flash_attention_2 --rope_method linear --rope_factor 4
 
-############## NTK ##############
+################ NTK ################
 torchrun --nproc_per_node 8 -m main.eval_longbench --data_root $data_root --max_length 15500 --use_flash_attention_2 --rope_method dynamic --rope_factor 2
 
-############## LongLlama ##############
+################ LongLlama ################
 torchrun --nproc_per_node 8 -m main.eval_longbench --data_root $data_root --max_length 15500 --model_name_or_path syzymon/long_llama_code_7b_instruct
 
-############## LongChat ##############
+################ LongChat ################
 torchrun --nproc_per_node 8 -m main.eval_longbench --data_root $data_root --max_length 31500 --model_name_or_path lmsys/longchat-7b-v1.5-32k --use_flash_attention_2
 
-############## Activation Beacon ##############
+################ Activation Beacon ################
 torchrun --nproc_per_node 8 -m main.eval_longbench --data_root $data_root --max_length 15500 --model_name_or_path namespace-Pt/activation-beacon-llama2-7b-chat
 ```
 
@@ -103,23 +105,52 @@ The results can be found at `data/results/longbench/metrics.log`.
 ```bash
 data_root="/data/activation-beacon"
 
-############## Llama-2 ##############
+################ Llama-2 ################
 torchrun --nproc_per_node 8 -m main.eval_longeval --data_root $data_root --use_flash_attention_2
 
-############## PI ##############
+################ PI ################
 torchrun --nproc_per_node 8 -m main.eval_longeval --data_root $data_root --use_flash_attention_2 --rope_method linear --rope_factor 4
 
-############## NTK ##############
+################ NTK ################
 torchrun --nproc_per_node 8 -m main.eval_longeval --data_root $data_root --use_flash_attention_2 --rope_method dynamic --rope_factor 2
 
-############## LongLlama ##############
+################ LongLlama ################
 torchrun --nproc_per_node 8 -m main.eval_longeval --data_root $data_root --model_name_or_path syzymon/long_llama_code_7b_instruct
 
-############## LongChat ##############
+################ LongChat ################
 torchrun --nproc_per_node 8 -m main.eval_longeval --data_root $data_root --model_name_or_path lmsys/longchat-7b-v1.5-32k --use_flash_attention_2
 
-############## Activation Beacon ##############
+################ Activation Beacon ################
 torchrun --nproc_per_node 8 -m main.eval_longeval --data_root $data_root --model_name_or_path namespace-Pt/activation-beacon-llama2-7b-chat
 ```
 
 The results can be found at `data/results/longeval/topic_retrieval.log`.
+
+
+### Passkey Retrieval
+```bash
+data_root="/data/activation-beacon"
+
+################ Llama-2 ################
+python -m main.eval_passkey --data_root $data_root --use_flash_attention_2
+
+################ PI ################
+python -m main.eval_passkey --data_root $data_root --use_flash_attention_2 --rope_method linear --rope_factor 4
+
+################ NTK ################
+python -m main.eval_passkey --data_root $data_root --use_flash_attention_2 --rope_method dynamic --rope_factor 2
+
+################ LongLlama ################
+python -m main.eval_passkey --data_root $data_root --model_name_or_path syzymon/long_llama_code_7b_instruct
+
+################ LongChat ################
+python -m main.eval_passkey --data_root $data_root --model_name_or_path lmsys/longchat-7b-v1.5-32k --use_flash_attention_2
+
+################ Activation Beacon ################
+python -m main.eval_passkey --data_root $data_root --model_name_or_path namespace-Pt/activation-beacon-llama2-7b-chat
+
+# enable retrieval to improve memory accuracy (still in progress)
+python -m main.eval_passkey --data_root $data_root --model_name_or_path namespace-Pt/activation-beacon-llama2-7b-chat --enable_beacon --retrieval_method bm25 --retrieval_topk 3 --beacon_ratio 2 128
+```
+
+The results can be found at `data/results/passkey/metrics.log`.