-
Notifications
You must be signed in to change notification settings
- Fork 155
Expand file tree
/
Copy pathgen_judgment.py
More file actions
201 lines (158 loc) · 6.16 KB
/
Copy pathgen_judgment.py
File metadata and controls
201 lines (158 loc) · 6.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
import json
import yaml
import argparse
import os
import concurrent.futures
from tqdm import tqdm
from utils.completion import (
load_questions,
registered_api_completion,
load_questions,
load_model_answers,
get_endpoint,
make_config,
)
from utils.judge_utils import JUDGE_SETTINGS
def get_score(judgment, patterns):
import re
for pattern in patterns:
pattern = re.compile(pattern)
matches = pattern.findall(judgment.upper())
matches = [m for m in matches if m != ""]
if len(set(matches)) > 0:
return matches[-1].strip("\n")
return None
def pairwise_judgment(question, baseline, answer, reference, configs, settings):
prompt_args = {
"QUESTION": question['prompt'],
"ANSWER_A": baseline["messages"][-1]["content"]['answer'],
"ANSWER_B": answer["messages"][-1]["content"]['answer'],
}
if reference:
prompt_args[f"REFERENCE"] = reference["messages"][-1]["content"]['answer']
user_prompt = configs["prompt_template"].format(**prompt_args)
messages = [
{
"role": "system",
"content": JUDGE_SETTINGS[question["category"]]["system_prompt"],
},
{
"role": "user",
"content": user_prompt,
}
]
# build arguments for api completions
kwargs = settings | {
"api_dict": get_endpoint(settings["endpoints"]),
"messages": messages,
}
kwargs['temperature'] = configs['temperature']
kwargs['max_tokens'] = configs['max_tokens']
api_completion_func = registered_api_completion[settings["api_type"]]
output = api_completion_func(**kwargs)
if output is None:
return None
score = get_score(output['answer'], configs["regex_patterns"])
result = {
"score": score,
"judgment": output,
"prompt": messages,
}
return result
def judgment(args):
answer = args['answer']
baseline = args['baseline']
output = {
"uid": args['question']["uid"],
"category": args['question']["category"],
"judge": args['configs']['judge_model'],
"model": answer["model"],
"baseline": baseline["model"],
"games": []
}
# round 1
result = pairwise_judgment(
question=args['question'],
baseline=baseline,
answer=answer,
reference=args['reference'],
configs=args['configs'],
settings=args['settings'],
)
output["games"].append(result)
# round 2
result = pairwise_judgment(
question=args['question'],
baseline=answer,
answer=baseline,
reference=args['reference'],
configs=args['configs'],
settings=args['settings'],
)
output["games"].append(result)
with open(args['output_file'], "a", encoding="utf-8") as f:
f.write(json.dumps(output, ensure_ascii=False) + "\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--setting-file", type=str, default="config/arena-hard-v2.0.yaml")
parser.add_argument("--endpoint-file", type=str, default="config/api_config.yaml")
args = parser.parse_args()
print(args)
configs = make_config(args.setting_file)
endpoint_list = make_config(args.endpoint_file)
print(f'judge model: {configs["judge_model"]}, reference: {configs["reference"]}, temperature: {configs["temperature"]}, max tokens: {configs["max_tokens"]}')
question_file = os.path.join("data", configs["bench_name"], "question.jsonl")
answer_dir = os.path.join("data", configs["bench_name"], "model_answer")
questions = load_questions(question_file)
model_answers = load_model_answers(answer_dir)
# if user choose a set of models, only judge those models
models = [model for model in configs["model_list"]]
if configs["reference"]:
assert not configs["reference"] in models, "ERROR: one of the models being evaluated is used as reference."
ref_answers = [answer_dir[model] for model in configs["reference"]]
else:
ref_answers = None
output_files = {}
output_dir = f"data/{configs['bench_name']}/model_judgment/{configs['judge_model']}"
for model in models:
output_files[model] = os.path.join(
output_dir,
f"{model}.jsonl",
)
for output_file in output_files.values():
os.makedirs(os.path.dirname(output_file), exist_ok=True)
existing_judgments = load_model_answers(output_dir)
endpoint_settings = endpoint_list[configs["judge_model"]]
with concurrent.futures.ThreadPoolExecutor(max_workers=endpoint_settings["parallel"]) as executor:
futures = []
for model in models:
count = 0
for question in questions:
uid = question["uid"]
kwargs = {}
kwargs["question"] = question
if model in model_answers and not uid in model_answers[model]:
print(f"Warning: {model} answer to {question['uid']} cannot be found.")
continue
if model in existing_judgments and uid in existing_judgments[model]:
count += 1
continue
kwargs["answer"] = model_answers[model][uid]
kwargs["baseline"] = model_answers[
JUDGE_SETTINGS[question["category"]]["baseline"]
][uid]
if ref_answers:
kwargs["reference"] = [ref_answer[uid] for ref_answer in ref_answers]
else:
kwargs["reference"] = None
kwargs["configs"] = configs
kwargs["settings"] = endpoint_settings
kwargs["output_file"] = output_files[model]
future = executor.submit(judgment, kwargs)
futures.append(future)
if count > 0:
print(f"{count} number of existing judgments")
for future in tqdm(
concurrent.futures.as_completed(futures), total=len(futures)
):
future.result()