change file in tests

PaddlePaddle · May 20, 2024 · 4e7edef · 4e7edef
1 parent 9ac0c8f
commit 4e7edef
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 30 deletions.
diff --git a/llm/docs/pretrain.rst b/llm/docs/pretrain.rst
@@ -44,8 +44,8 @@ git clone 代码到本地，即可开始。
 .. code-block:: bash
 
     # llama 模型数据下载
-    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
     # gpt 模型数据下载
     # wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_ids.npy
@@ -57,8 +57,8 @@ git clone 代码到本地，即可开始。
 .. code-block:: bash
 
     mkdir data
-    mv llama_openwebtext_100k_ids.npy ./data
-    mv llama_openwebtext_100k_idx.npz ./data
+    mv llama_openwebtext_100k.bin ./data
+    mv llama_openwebtext_100k.idx ./data
 
 
 

diff --git a/scripts/distribute/ci_case_auto.sh b/scripts/distribute/ci_case_auto.sh
@@ -2150,8 +2150,8 @@ function before_hook_for_llama() {
         else
             # download data for llama
             mkdir ${llama_data_path}/data;
-            wget -O ${llama_data_path}/data/llama_openwebtext_100k_ids.npy https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy;
-            wget -O ${llama_data_path}/data/llama_openwebtext_100k_idx.npz https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz;
+            wget -O ${llama_data_path}/data/llama_openwebtext_100k.bin https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin;
+            wget -O ${llama_data_path}/data/llama_openwebtext_100k.idx https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx;
         fi
         cp -r ${llama_data_path}/data ${llama_case_path}/
     else

diff --git a/tests/llm/test_pretrain.py b/tests/llm/test_pretrain.py
@@ -59,8 +59,8 @@ def test_pretrain(self):
             del sys.modules["run_pretrain"]
 
         # Run pretrain
-        URL = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy"
-        URL2 = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz"
+        URL = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin"
+        URL2 = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx"
         get_path_from_url(URL, root_dir=self.dataset_dir)
         get_path_from_url(URL2, root_dir=self.dataset_dir)
 

diff --git a/tests/test_tipc/auto_tuner/llama_pretrain/benchmark_common/prepare.sh b/tests/test_tipc/auto_tuner/llama_pretrain/benchmark_common/prepare.sh
@@ -24,11 +24,11 @@ cd ../../../llm/llama
 python -m pip install tool_helpers
 
 rm -rf data && mkdir data
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
-mv llama_openwebtext_100k_ids.npy ./data
-mv llama_openwebtext_100k_idx.npz ./data
+mv llama_openwebtext_100k.bin ./data
+mv llama_openwebtext_100k.idx ./data
 
 # mv autoconfig
 rm -rf autoconfig

diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh b/tests/test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
@@ -23,9 +23,9 @@ python setup.py install
 cd ../../../llm/llama
 python -m pip install tool_helpers
 
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
 mkdir data
-mv llama_openwebtext_100k_ids.npy ./data
-mv llama_openwebtext_100k_idx.npz ./data
+mv llama_openwebtext_100k.bin ./data
+mv llama_openwebtext_100k.idx ./data
diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/benchmark_common/prepare.sh b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/benchmark_common/prepare.sh
@@ -24,11 +24,11 @@ cd ../../../llm/llama
 python -m pip install tool_helpers
 
 # download data
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 mkdir data
-mv llama_openwebtext_100k_ids.npy ./data
-mv llama_openwebtext_100k_idx.npz ./data
+mv llama_openwebtext_100k.bin ./data
+mv llama_openwebtext_100k.idx ./data
 
 # install etcd
 pip install httpx etcd3 protobuf==3.20.0 --force-reinstall

diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/qwen/benchmark_common/prepare.sh b/tests/test_tipc/dygraph/hybrid_parallelism/qwen/benchmark_common/prepare.sh
@@ -25,9 +25,9 @@ python setup.py install
 cd ../../../llm/qwen
 python -m pip install tool_helpers
 
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
 mkdir data
-mv llama_openwebtext_100k_ids.npy ./data
-mv llama_openwebtext_100k_idx.npz ./data
+mv llama_openwebtext_100k.bin ./data
+mv llama_openwebtext_100k.idx ./data
diff --git a/tests/trainer/test_unified_checkpoint.py b/tests/trainer/test_unified_checkpoint.py
@@ -186,8 +186,8 @@ def setUp(self):
         os.environ.update(environment_variables)
 
         files = [
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy",
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx",
         ]
         self.prepare_inputs_data(pretrain_arguments["input_dir"], files)
 
@@ -646,8 +646,8 @@ def setUp(self):
         os.environ.update(environment_variables)
 
         files = [
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy",
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx",
         ]
         self.prepare_inputs_data(pretrain_arguments["input_dir"], files)
 
@@ -687,8 +687,8 @@ def setUp(self):
         os.environ.update(environment_variables)
 
         files = [
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy",
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx",
         ]
         self.prepare_inputs_data(pretrain_arguments["input_dir"], files)