forked from masa3141/japanese-alpaca-lora
-
Notifications
You must be signed in to change notification settings - Fork 0
/
eval_calm.py
77 lines (60 loc) · 2.03 KB
/
eval_calm.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,4,5,6,7,8,9"
from datasets import load_dataset
from transformers import GenerationConfig, AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
model_custom = AutoModelForCausalLM.from_pretrained("cyberagent/open-calm-7b", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained("cyberagent/open-calm-7b")
tokenizer.pad_token_id = 0
model_custom = PeftModel.from_pretrained(model_custom, "izumi-lab/stormy-7b-10ep")
model_custom = model_custom.merge_and_unload()
test_data = load_dataset('json', data_files='test_data.json')
test_data = test_data['train']
def generate_prompt(data_point):
if data_point["input"]:
return f"""
### Instruction:
{data_point["instruction"]}
### Input:
{data_point["input"]}
### Response:
"""
else:
return f"""
### Instruction:
{data_point["instruction"]}
### Response:
"""
generation_config = GenerationConfig(
temperature=0.1,
top_p=0.75,
num_beams=4,
)
def evaluate(model_aaa, test_data):
prompt = generate_prompt(test_data)
inputs = tokenizer(prompt, return_tensors="pt")
generation_output = model_aaa.generate(
input_ids=inputs['input_ids'].to('cuda'),
generation_config=generation_config,
return_dict_in_generate=True,
output_scores=True,
max_new_tokens=256,
num_beams=5,
no_repeat_ngram_size=2,
early_stopping=True
)
# with open('log.txt', 'w') as f:
# for s in generation_output.sequences:
# output = tokenizer.decode(s)
# f.write(output.split("### Response:")[1].strip() + '\n')
for s in generation_output.sequences:
output = tokenizer.decode(s)
print("Response:", output)
return output
# evaluate(model_custom, test_data)
with open('Calm_instruct_tuning.txt', 'w') as f:
for i in test_data:
temp = evaluate(model_custom, i)
f.write(temp + '\n')
f.close()
print('生成结束。')