1036 lines
42 KiB
Python
1036 lines
42 KiB
Python
|
# Copyright 2023 The OPRO Authors
|
|||
|
|
#
|
|||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|||
|
|
# you may not use this file except in compliance with the License.
|
|||
|
|
# You may obtain a copy of the License at
|
|||
|
|
#
|
|||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
#
|
|||
|
|
# Unless required by applicable law or agreed to in writing, software
|
|||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
# See the License for the specific language governing permissions and
|
|||
|
|
# limitations under the License.
|
|||
|
|
"""The utility functions for prompt optimization."""
|
|||
|
|
|
|||
|
|
import collections
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import pickle
|
|||
|
|
import re
|
|||
|
|
import sys
|
|||
|
|
|
|||
|
|
OPRO_ROOT_PATH = os.path.dirname(
|
|||
|
|
os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
|||
|
|
)
|
|||
|
|
sys.path.insert(0, OPRO_ROOT_PATH)
|
|||
|
|
|
|||
|
|
import numpy as np
|
|||
|
|
from opro.evaluation import eval_utils
|
|||
|
|
import pandas as pd
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_string_in_square_brackets(input_string):
|
|||
|
|
raw_result = re.findall(r"\[.*?\]", input_string)
|
|||
|
|
if raw_result:
|
|||
|
|
return raw_result[0][1:-1]
|
|||
|
|
else:
|
|||
|
|
return "" #从输入的字符串 input_string中提取第一个被方括号 []包裹的内容，并返回去掉方括号后的结果。如果输入字符串中没有方括号，则返回空字符串 ""。
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_tag_content(text, prefix="<TEXT>", suffix="</TEXT>"):
|
|||
|
|
pattern = f"{prefix}(.*?){suffix}"
|
|||
|
|
results = re.findall(pattern, text, re.DOTALL)
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _bucketize_float(num, n_buckets=20):
|
|||
|
|
assert num >= 0 and num <= 1, "The given number must be between 0 and 1."
|
|||
|
|
return round(num * n_buckets) #此函数可能用于将指令的评分（如 0.85）转换为整数分桶
|
|||
|
|
|
|||
|
|
|
|||
|
|
def gen_ins_and_score_pairs_substr(
|
|||
|
|
old_instructions_and_scores,
|
|||
|
|
old_instruction_score_threshold=0.1,
|
|||
|
|
max_num_instructions=1000,
|
|||
|
|
return_str_only=False,
|
|||
|
|
num_score_buckets=np.inf,
|
|||
|
|
):
|
|||
|
|
"""Generate the string that includes instruction-score pairs."""
|
|||
|
|
assert num_score_buckets == np.inf or isinstance(num_score_buckets, int)
|
|||
|
|
old_instructions_and_scores_str = ""
|
|||
|
|
old_instructions_and_scores = sorted(
|
|||
|
|
old_instructions_and_scores, key=lambda x: x[1]
|
|||
|
|
)[-max_num_instructions:]
|
|||
|
|
old_instructions_and_scores_in_meta_prompt = []
|
|||
|
|
for instruction, score, i_step in old_instructions_and_scores:
|
|||
|
|
if (
|
|||
|
|
not old_instruction_score_threshold
|
|||
|
|
or score >= old_instruction_score_threshold
|
|||
|
|
):
|
|||
|
|
old_instructions_and_scores_in_meta_prompt.append(
|
|||
|
|
(instruction, score, i_step)
|
|||
|
|
)
|
|||
|
|
if num_score_buckets == np.inf:
|
|||
|
|
score_to_show = round(score, 3)
|
|||
|
|
else:
|
|||
|
|
score_to_show = _bucketize_float(score, num_score_buckets)
|
|||
|
|
old_instructions_and_scores_str += (
|
|||
|
|
f"\ntext:\n{instruction}\nscore:\n{score_to_show}\n"
|
|||
|
|
)
|
|||
|
|
if return_str_only:
|
|||
|
|
return old_instructions_and_scores_str
|
|||
|
|
else:
|
|||
|
|
return (
|
|||
|
|
old_instructions_and_scores_str,
|
|||
|
|
old_instructions_and_scores_in_meta_prompt,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def gen_meta_prompt(
|
|||
|
|
old_instructions_and_scores,
|
|||
|
|
instruction_pos,
|
|||
|
|
optimizer_llm_name,
|
|||
|
|
old_instruction_score_threshold=0.1,
|
|||
|
|
max_num_instructions=1000,
|
|||
|
|
meta_prompt_type="both_instructions_and_exemplars",
|
|||
|
|
few_shot_qa_pairs=False,
|
|||
|
|
include_qa=True,
|
|||
|
|
data=None,
|
|||
|
|
few_shot_index_list=None,
|
|||
|
|
instructions_before_exemplars=True,
|
|||
|
|
num_score_buckets=np.inf,
|
|||
|
|
dataset_name="",
|
|||
|
|
task_name="",
|
|||
|
|
):
|
|||
|
|
"""Generate meta prompt for instruction rewriting.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
old_instructions_and_scores (list): a list of (instruction, score, i_step)
|
|||
|
|
pairs.
|
|||
|
|
instruction_pos (str): where to put the instruction, one of {'before_QA',
|
|||
|
|
'Q_begin', 'Q_end', 'A_begin'}.
|
|||
|
|
optimizer_llm_name (str): the name of the LLM used for instruction editing.
|
|||
|
|
old_instruction_score_threshold (float): only add old instructions with score
|
|||
|
|
no less than this threshold.
|
|||
|
|
max_num_instructions (int): the maximum number of instructions in the meta
|
|||
|
|
prompt.
|
|||
|
|
meta_prompt_type (str): the type of meta-prompt: whether to have both
|
|||
|
|
previous instructions and dataset exemplars (often for fine-tuned
|
|||
|
|
optimizers), or to have only previous instructions (often for pre-trained
|
|||
|
|
optimizers).
|
|||
|
|
few_shot_qa_pairs (bool): whether to have few-shot QA pairs in the meta
|
|||
|
|
prompt.
|
|||
|
|
include_qa (bool): whether to include "Q:" and "A:" formats in the prompt.
|
|||
|
|
data (list or pd.DataFrame): the raw data.
|
|||
|
|
few_shot_index_list (list): the list of indices of few-shot examples.
|
|||
|
|
instructions_before_exemplars (bool): whether the instruction-score pairs are
|
|||
|
|
before the exemplars from the dataset.
|
|||
|
|
num_score_buckets (np.inf or int): the number of score buckets when we
|
|||
|
|
convert float accuracies to integers. Default to np.inf for not
|
|||
|
|
bucketizing.
|
|||
|
|
dataset_name (str): the name of the current dataset. Only used when
|
|||
|
|
generating task description when meta_prompt_type == "instructions_only".
|
|||
|
|
task_name (str): the name of the current task. Only used when generating task
|
|||
|
|
description when meta_prompt_type == "instructions_only".
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
meta_prompt (str): the generated meta prompt.
|
|||
|
|
"""
|
|||
|
|
assert instruction_pos in {
|
|||
|
|
"before_Q",
|
|||
|
|
"Q_begin",
|
|||
|
|
"Q_end",
|
|||
|
|
"A_begin",
|
|||
|
|
}, (
|
|||
|
|
"The instruction position should be either before the question, or at the"
|
|||
|
|
" beginning of the question, at the end of the question, or at the"
|
|||
|
|
" beginning of the answer."
|
|||
|
|
)
|
|||
|
|
assert meta_prompt_type in {
|
|||
|
|
"both_instructions_and_exemplars",
|
|||
|
|
"instructions_only",
|
|||
|
|
}
|
|||
|
|
assert dataset_name in {
|
|||
|
|
"mmlu",
|
|||
|
|
"bbh",
|
|||
|
|
"gsm8k",
|
|||
|
|
}, "The lower-case dataset name must be one of mmlu, bbh, gsm8k."
|
|||
|
|
assert num_score_buckets == np.inf or isinstance(num_score_buckets, int)
|
|||
|
|
|
|||
|
|
meta_prompt = ""
|
|||
|
|
if meta_prompt_type == "both_instructions_and_exemplars":
|
|||
|
|
if optimizer_llm_name.lower() in {"gpt-3.5-turbo", "gpt-4","local"}: # GPT模型的提示模板
|
|||
|
|
if instruction_pos == "A_begin":# 针对答案开头部分的描述
|
|||
|
|
meta_prompt_old_instruction_part = (
|
|||
|
|
"Your task is to generate the answer starting sentence <Start>."
|
|||
|
|
" Below are some previous starting sentences with their scores."
|
|||
|
|
" The score ranges from 0 to 100.\n"
|
|||
|
|
)#要求模型生成答案的起始句（如“The answer is...”），并参考历史评分数据。
|
|||
|
|
else:# 针对普通指令的描述
|
|||
|
|
meta_prompt_old_instruction_part = (
|
|||
|
|
"Your task is to generate the instruction <INS>."
|
|||
|
|
" Below are some previous instructions with their scores."
|
|||
|
|
" The score ranges from 0 to 100.\n"
|
|||
|
|
)
|
|||
|
|
else: # text-bison 模型的专用提示模板
|
|||
|
|
assert optimizer_llm_name.lower() == "text-bison"
|
|||
|
|
meta_prompt_old_instruction_part = (
|
|||
|
|
"I have some texts along with their corresponding scores."
|
|||
|
|
" The texts are arranged in ascending order based on their scores,"
|
|||
|
|
" where higher scores indicate better quality.\n\n"
|
|||
|
|
)
|
|||
|
|
# add old instructions
|
|||
|
|
old_instructions_and_scores_str = gen_ins_and_score_pairs_substr(
|
|||
|
|
old_instructions_and_scores=old_instructions_and_scores,
|
|||
|
|
old_instruction_score_threshold=old_instruction_score_threshold,
|
|||
|
|
max_num_instructions=max_num_instructions,
|
|||
|
|
return_str_only=True,
|
|||
|
|
num_score_buckets=num_score_buckets,
|
|||
|
|
)
|
|||
|
|
meta_prompt_old_instruction_part += old_instructions_and_scores_str
|
|||
|
|
# add QA pairs if few_shot_qa_pairs == True
|
|||
|
|
meta_prompt_exemplar_part = ""
|
|||
|
|
if few_shot_qa_pairs:
|
|||
|
|
if optimizer_llm_name.lower() in {"gpt-3.5-turbo", "gpt-4", "local"}:
|
|||
|
|
meta_prompt_exemplar_part += "Below are some problems.\n"
|
|||
|
|
else:
|
|||
|
|
assert optimizer_llm_name.lower() == "text-bison"
|
|||
|
|
meta_prompt_exemplar_part += (
|
|||
|
|
"The following exemplars show how to apply your text: you replace"
|
|||
|
|
" <INS> in each input with your text, then read the input and give"
|
|||
|
|
" an output. We say your output is wrong if your output is"
|
|||
|
|
" different from the given output, and we say your output is"
|
|||
|
|
" correct if they are the same. When replacing <INS> with an old"
|
|||
|
|
" piece of text above, we get wrong outputs on the following"
|
|||
|
|
" inputs.\n\n"
|
|||
|
|
)
|
|||
|
|
for idx in few_shot_index_list:
|
|||
|
|
if dataset_name == "mmlu":
|
|||
|
|
question = eval_utils._format_mmlu_example(data, idx) # pylint: disable=protected-access
|
|||
|
|
true_answer = data.iloc[idx, -1]
|
|||
|
|
elif dataset_name == "bbh":
|
|||
|
|
question = data[idx]["input"]
|
|||
|
|
true_answer = data[idx]["target"]
|
|||
|
|
else:
|
|||
|
|
assert dataset_name == "gsm8k"
|
|||
|
|
question = data.iloc[idx, 0]
|
|||
|
|
true_answer = data.iloc[idx, 1]
|
|||
|
|
|
|||
|
|
if include_qa: # when "Q:" and "A:" are present in the prompt
|
|||
|
|
if instruction_pos == "before_Q":
|
|||
|
|
meta_prompt_exemplar_part += f"\ninput:\n<INS>\nQ: {question}\nA:"
|
|||
|
|
elif instruction_pos == "Q_begin":
|
|||
|
|
meta_prompt_exemplar_part += f"\ninput:\nQ: <INS>\n{question}\nA:"
|
|||
|
|
elif instruction_pos == "Q_end":
|
|||
|
|
meta_prompt_exemplar_part += f"\ninput:\nQ: {question}\n<INS>\nA:"
|
|||
|
|
else: # instruction_pos == "A_begin"
|
|||
|
|
if optimizer_llm_name.lower() in {"gpt-3.5-turbo", "gpt-4", "local"}:
|
|||
|
|
meta_prompt_exemplar_part += f"\nQ: {question}\nA: <Start>"
|
|||
|
|
else:
|
|||
|
|
assert optimizer_llm_name.lower() == "text-bison"
|
|||
|
|
meta_prompt_exemplar_part += f"\ninput:\nQ: {question}\nA: <INS>"
|
|||
|
|
else: # when there're no "Q:" and "A:" in the prompt
|
|||
|
|
assert instruction_pos in {"Q_begin", "Q_end"}
|
|||
|
|
if optimizer_llm_name.lower() in {"gpt-3.5-turbo", "gpt-4", "local"}:
|
|||
|
|
if instruction_pos == "Q_begin":
|
|||
|
|
meta_prompt_exemplar_part += f"\nProblem:\n<INS>\n{question}\n"
|
|||
|
|
elif instruction_pos == "Q_end":
|
|||
|
|
meta_prompt_exemplar_part += f"\nProblem:\n{question}\n<INS>\n"
|
|||
|
|
else:
|
|||
|
|
assert optimizer_llm_name.lower() == "text-bison"
|
|||
|
|
if instruction_pos == "Q_begin":
|
|||
|
|
meta_prompt_exemplar_part += f"\ninput:\n<INS>\n{question}\n"
|
|||
|
|
elif instruction_pos == "Q_end":
|
|||
|
|
meta_prompt_exemplar_part += f"\ninput:\n{question}\n<INS>\n"
|
|||
|
|
|
|||
|
|
if optimizer_llm_name.lower() in {"gpt-3.5-turbo", "gpt-4", "local"}:
|
|||
|
|
meta_prompt_exemplar_part += (
|
|||
|
|
f"\nGround truth answer:\n{true_answer}\n"
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
assert optimizer_llm_name.lower() == "text-bison"
|
|||
|
|
meta_prompt_exemplar_part += f"\noutput:\n{true_answer}\n"
|
|||
|
|
|
|||
|
|
if few_shot_qa_pairs:
|
|||
|
|
if instructions_before_exemplars:
|
|||
|
|
meta_prompt += (
|
|||
|
|
meta_prompt_old_instruction_part
|
|||
|
|
+ "\n\n"
|
|||
|
|
+ meta_prompt_exemplar_part
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
meta_prompt += (
|
|||
|
|
meta_prompt_exemplar_part
|
|||
|
|
+ "\n\n"
|
|||
|
|
+ meta_prompt_old_instruction_part
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
meta_prompt += meta_prompt_old_instruction_part
|
|||
|
|
|
|||
|
|
if optimizer_llm_name.lower() in {"gpt-3.5-turbo", "gpt-4", "local"}:
|
|||
|
|
if instruction_pos == "A_begin":
|
|||
|
|
meta_prompt += (
|
|||
|
|
"\n\nGenerate a starting sentence that is different from all the"
|
|||
|
|
" <Start> sentences above, and has a higher score than all the"
|
|||
|
|
" <Start> sentences above. The starting sentence should begin with"
|
|||
|
|
" <Start> and end with </Start>. The starting sentence should be"
|
|||
|
|
" concise, effective, and generally applicable to all QA pairs"
|
|||
|
|
" above."
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
meta_prompt += (
|
|||
|
|
"\n\nGenerate an instruction that"
|
|||
|
|
" is different from all the instructions <INS> above,"
|
|||
|
|
" and has a higher score than all the instructions <INS> above."
|
|||
|
|
" The instruction should begin with <INS> and end with </INS>."
|
|||
|
|
" The instruction should be concise, effective,"
|
|||
|
|
" and generally applicable to all problems above."
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
assert optimizer_llm_name.lower() == "text-bison"
|
|||
|
|
meta_prompt += (
|
|||
|
|
"\n\nWrite your new text that is different from the old ones and"
|
|||
|
|
" has a score as high as possible. Write the text in square brackets."
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
# when using a pre-trained model as optimizer
|
|||
|
|
assert meta_prompt_type == "instructions_only"
|
|||
|
|
|
|||
|
|
assert instruction_pos in {"Q_begin", "Q_end", "A_begin"}
|
|||
|
|
if instruction_pos == "Q_begin":
|
|||
|
|
instruction_pos_description = "at the beginning of the question"
|
|||
|
|
elif instruction_pos == "Q_end":
|
|||
|
|
instruction_pos_description = "at the end of the question"
|
|||
|
|
else:
|
|||
|
|
assert instruction_pos == "A_begin"
|
|||
|
|
instruction_pos_description = "at the beginning of the answer"
|
|||
|
|
|
|||
|
|
if dataset_name == "gsm8k":
|
|||
|
|
instruction_task_description = "grade school math"
|
|||
|
|
elif dataset_name == "mmlu":
|
|||
|
|
instruction_task_description = task_name
|
|||
|
|
else:
|
|||
|
|
assert dataset_name == "bbh"
|
|||
|
|
instruction_task_description = " ".join(task_name.split("_"))
|
|||
|
|
|
|||
|
|
meta_instruction = (
|
|||
|
|
f"Create a piece of text {instruction_pos_description.strip()} to"
|
|||
|
|
" enhance the precision in solving diverse"
|
|||
|
|
f" {instruction_task_description.strip()} problems."
|
|||
|
|
)
|
|||
|
|
old_instructions_and_scores = sorted(
|
|||
|
|
old_instructions_and_scores, key=lambda x: x[1]
|
|||
|
|
)
|
|||
|
|
old_instructions_and_scores_str = ""
|
|||
|
|
for instruction, score, _ in old_instructions_and_scores:
|
|||
|
|
if num_score_buckets == np.inf:
|
|||
|
|
score_to_show = round(score, 2)
|
|||
|
|
else:
|
|||
|
|
score_to_show = _bucketize_float(score, num_score_buckets)
|
|||
|
|
old_instructions_and_scores_str += (
|
|||
|
|
f"\n\nPrecision: {score_to_show} <TEXT>{instruction}</TEXT>"
|
|||
|
|
)
|
|||
|
|
meta_prompt += meta_instruction + old_instructions_and_scores_str
|
|||
|
|
return meta_prompt
|
|||
|
|
|
|||
|
|
|
|||
|
|
def run_evolution(**kwargs):
|
|||
|
|
"""The function for evolution."""
|
|||
|
|
# ================= experiment configurations =============================
|
|||
|
|
num_search_steps = kwargs["num_search_steps"]
|
|||
|
|
old_instruction_score_threshold = kwargs["old_instruction_score_threshold"]
|
|||
|
|
scorer_llm_dict = kwargs["scorer_llm_dict"]
|
|||
|
|
optimizer_llm_dict = kwargs["optimizer_llm_dict"]
|
|||
|
|
extract_final_answer_by_prompting_again = kwargs[
|
|||
|
|
"extract_final_answer_by_prompting_again"
|
|||
|
|
]
|
|||
|
|
include_qa = kwargs["include_qa"]
|
|||
|
|
evaluate_in_parallel = kwargs["evaluate_in_parallel"]
|
|||
|
|
tasks_all = kwargs["tasks_all"]
|
|||
|
|
train_ratio = kwargs["train_ratio"]
|
|||
|
|
eval_ratio = kwargs["eval_ratio"]
|
|||
|
|
test_ratio = kwargs["test_ratio"]
|
|||
|
|
train_index = kwargs["train_index"]
|
|||
|
|
eval_index = kwargs["eval_index"]
|
|||
|
|
dataset_name = kwargs["dataset_name"]
|
|||
|
|
task_name = kwargs["task_name"]
|
|||
|
|
num_examples = kwargs["num_examples"]
|
|||
|
|
root_data_folder_path = kwargs["root_data_folder_path"]
|
|||
|
|
optimizer_llm_temperature = kwargs["optimizer_llm_temperature"]
|
|||
|
|
optimizer_llm_temperature_schedule = (
|
|||
|
|
kwargs["optimizer_llm_temperature_schedule"]
|
|||
|
|
if "optimizer_llm_temperature_schedule" in kwargs
|
|||
|
|
else "constant"
|
|||
|
|
)
|
|||
|
|
optimizer_llm_temperature_end = (
|
|||
|
|
kwargs["optimizer_llm_temperature_end"]
|
|||
|
|
if "optimizer_llm_temperature_end" in kwargs
|
|||
|
|
else None
|
|||
|
|
)
|
|||
|
|
initial_instructions = kwargs["initial_instructions"]
|
|||
|
|
multiple_choice_tasks = kwargs["multiple_choice_tasks"]
|
|||
|
|
raw_data = kwargs["raw_data"]
|
|||
|
|
call_scorer_server_func = kwargs["call_scorer_server_func"]
|
|||
|
|
call_optimizer_server_func = kwargs["call_optimizer_server_func"]
|
|||
|
|
instruction_pos = kwargs["instruction_pos"]
|
|||
|
|
prediction_treat_as_number = kwargs["prediction_treat_as_number"]
|
|||
|
|
prediction_treat_as_bool = kwargs["prediction_treat_as_bool"]
|
|||
|
|
result_by_instruction_folder = kwargs["result_by_instruction_folder"]
|
|||
|
|
few_shot_qa_pairs = kwargs["few_shot_qa_pairs"]
|
|||
|
|
num_score_buckets = kwargs["num_score_buckets"]
|
|||
|
|
max_num_instructions = kwargs["max_num_instructions"]
|
|||
|
|
meta_prompt_type = kwargs["meta_prompt_type"]
|
|||
|
|
meta_prompt_instructions_before_exemplars = kwargs[
|
|||
|
|
"meta_prompt_instructions_before_exemplars"
|
|||
|
|
]
|
|||
|
|
few_shot_selection_criteria = kwargs["few_shot_selection_criteria"]
|
|||
|
|
optimizer_llm_name = kwargs["optimizer_llm_name"]
|
|||
|
|
num_generated_instructions_in_each_step = kwargs[
|
|||
|
|
"num_generated_instructions_in_each_step"
|
|||
|
|
]
|
|||
|
|
evaluate_generated_ins_on_few_shot = kwargs[
|
|||
|
|
"evaluate_generated_ins_on_few_shot"
|
|||
|
|
]
|
|||
|
|
num_few_shot_questions_for_instruction_refinement = kwargs[
|
|||
|
|
"num_few_shot_questions_for_instruction_refinement"
|
|||
|
|
]
|
|||
|
|
evaluate_old_ins_on_few_shot = kwargs["evaluate_old_ins_on_few_shot"]
|
|||
|
|
eval_interval = kwargs["eval_interval"]
|
|||
|
|
save_folder = kwargs["save_folder"]
|
|||
|
|
verbose = kwargs["verbose"] if "verbose" in kwargs else False
|
|||
|
|
|
|||
|
|
# =================== assertions =====================
|
|||
|
|
assert dataset_name in {
|
|||
|
|
"mmlu",
|
|||
|
|
"bbh",
|
|||
|
|
"gsm8k",
|
|||
|
|
}, "The lower-case dataset name must be one of mmlu, bbh, gsm8k."
|
|||
|
|
assert optimizer_llm_temperature_schedule in {
|
|||
|
|
"constant",
|
|||
|
|
"linear_increase",
|
|||
|
|
}, "The temperature schedule should be constant or linear_increase."
|
|||
|
|
|
|||
|
|
# =================== save configurations to json file ====================
|
|||
|
|
configs_dict = dict()
|
|||
|
|
configs_dict["scorer_llm_dict"] = scorer_llm_dict
|
|||
|
|
configs_dict["optimizer_llm_dict"] = optimizer_llm_dict
|
|||
|
|
configs_dict["instruction_pos"] = instruction_pos
|
|||
|
|
configs_dict["optimizer_llm_temperature"] = optimizer_llm_temperature
|
|||
|
|
configs_dict["optimizer_llm_temperature_schedule"] = (
|
|||
|
|
optimizer_llm_temperature_schedule
|
|||
|
|
)
|
|||
|
|
configs_dict["optimizer_llm_temperature_end"] = optimizer_llm_temperature_end
|
|||
|
|
with open(os.path.join(save_folder, "configs_dict.json"), "w") as f:
|
|||
|
|
json.dump(configs_dict, f, indent=4)
|
|||
|
|
|
|||
|
|
num_servers = scorer_llm_dict["num_servers"]
|
|||
|
|
batch_size = scorer_llm_dict["batch_size"]
|
|||
|
|
generated_ins_on_few_shot_results_dict = dict()
|
|||
|
|
old_ins_on_few_shot_results_dict = dict()
|
|||
|
|
# evaluation results every a few steps
|
|||
|
|
# format: [(i_step, instruction, detailed_results_df)]
|
|||
|
|
eval_results = []
|
|||
|
|
# all generated instructions, format: [(instruction, score, step_index)]
|
|||
|
|
# the instructions that were skipped have score NaN
|
|||
|
|
old_instructions_and_scores_raw = []
|
|||
|
|
# the new instructions, format: [(instruction, score, step_index)]
|
|||
|
|
old_instructions_and_scores = []
|
|||
|
|
meta_prompts = [] # format: [(meta_prompt, step_index)]
|
|||
|
|
instruction_score_dict = dict() # the dictionary of {instruction: score}
|
|||
|
|
# the dictionary of the few-shot QA indices in meta-prompt
|
|||
|
|
# key: step index; value: the list of few-shot indices in that step
|
|||
|
|
few_shot_index_list_by_step_dict = dict()
|
|||
|
|
detailed_results_df_by_instruction_dict = dict()
|
|||
|
|
wrong_questions_from_start_counter = collections.Counter()
|
|||
|
|
# EVAL results
|
|||
|
|
eval_detailed_results_df_dict = dict() # {instruction: detailed_results_df}
|
|||
|
|
instruction_eval_score_dict = dict() # {instruction: eval_score}
|
|||
|
|
old_instruction_md5_hashstrings_set = set()
|
|||
|
|
|
|||
|
|
print(f"tasks_all: {tasks_all}")
|
|||
|
|
print(
|
|||
|
|
f"train_ratio: {train_ratio}, number of training points:"
|
|||
|
|
f" {int(num_examples * train_ratio)}"
|
|||
|
|
)
|
|||
|
|
print(
|
|||
|
|
f"eval_ratio: {eval_ratio}, number of eval points: "
|
|||
|
|
f"{int(num_examples * eval_ratio)}"
|
|||
|
|
)
|
|||
|
|
print(
|
|||
|
|
f"test_ratio: {test_ratio}, number of test points: "
|
|||
|
|
f"{int(num_examples * test_ratio)}"
|
|||
|
|
)
|
|||
|
|
print(
|
|||
|
|
f"optimizer llm temperature: {optimizer_llm_temperature}, schedule:"
|
|||
|
|
f" {optimizer_llm_temperature_schedule}"
|
|||
|
|
)
|
|||
|
|
print(
|
|||
|
|
f"generating {num_generated_instructions_in_each_step} instructions in"
|
|||
|
|
f" each step, run for {num_search_steps} steps"
|
|||
|
|
)
|
|||
|
|
print(
|
|||
|
|
"discarding generated instructions with score less than:"
|
|||
|
|
f" {old_instruction_score_threshold} (old_instruction_score_threshold)"
|
|||
|
|
)
|
|||
|
|
print(f"num_score_buckets: {num_score_buckets}")
|
|||
|
|
|
|||
|
|
if dataset_name == "mmlu":
|
|||
|
|
is_multiple_choice = True
|
|||
|
|
is_multiple_choice_eval = True
|
|||
|
|
elif dataset_name in {"gsm8k"}:
|
|||
|
|
is_multiple_choice = False
|
|||
|
|
is_multiple_choice_eval = False
|
|||
|
|
else:
|
|||
|
|
assert dataset_name == "bbh"
|
|||
|
|
is_multiple_choice = []
|
|||
|
|
is_multiple_choice_eval = []
|
|||
|
|
train_index_by_task_dict = dict()
|
|||
|
|
eval_index_by_task_dict = dict()
|
|||
|
|
start_index = 0
|
|||
|
|
for task_name in tasks_all:
|
|||
|
|
single_task_list = eval_utils.load_bbh_task_data(
|
|||
|
|
task_name, base_dir=root_data_folder_path
|
|||
|
|
)
|
|||
|
|
end_index = start_index + len(single_task_list)
|
|||
|
|
train_index_by_task_dict[task_name] = (
|
|||
|
|
train_index[(train_index >= start_index) & (train_index < end_index)]
|
|||
|
|
# if " - start_index" is added here, then the dict would contain
|
|||
|
|
# indices in the original task
|
|||
|
|
)
|
|||
|
|
eval_index_by_task_dict[task_name] = (
|
|||
|
|
eval_index[(eval_index >= start_index) & (eval_index < end_index)]
|
|||
|
|
# if " - start_index" is added here, then the dict would contain
|
|||
|
|
# indices in the original task
|
|||
|
|
)
|
|||
|
|
start_index = end_index
|
|||
|
|
is_multiple_choice_single_task_train = [
|
|||
|
|
task_name in multiple_choice_tasks
|
|||
|
|
] * len(train_index_by_task_dict[task_name])
|
|||
|
|
is_multiple_choice_single_task_eval = [
|
|||
|
|
task_name in multiple_choice_tasks
|
|||
|
|
] * len(eval_index_by_task_dict[task_name])
|
|||
|
|
is_multiple_choice += is_multiple_choice_single_task_train
|
|||
|
|
is_multiple_choice_eval += is_multiple_choice_single_task_eval
|
|||
|
|
|
|||
|
|
prev_saved_instructions = set()
|
|||
|
|
|
|||
|
|
# evaluate initial instructions
|
|||
|
|
print("\n============== evaluating initial instructions ===============")
|
|||
|
|
for instruction in initial_instructions:
|
|||
|
|
print(f"""computing the score of "{instruction}" by prompting""")
|
|||
|
|
|
|||
|
|
detailed_results_df = eval_utils.evaluate_single_instruction(
|
|||
|
|
data=raw_data,
|
|||
|
|
instruction=instruction,
|
|||
|
|
eval_index_all=train_index,
|
|||
|
|
batch_size=batch_size,
|
|||
|
|
call_server_func=call_scorer_server_func,
|
|||
|
|
dataset_name=dataset_name,
|
|||
|
|
num_servers=num_servers,
|
|||
|
|
extract_final_answer_by_prompting_again=extract_final_answer_by_prompting_again,
|
|||
|
|
include_qa=include_qa,
|
|||
|
|
evaluate_in_parallel=evaluate_in_parallel,
|
|||
|
|
instruction_pos=instruction_pos,
|
|||
|
|
is_multiple_choice=is_multiple_choice,
|
|||
|
|
prediction_treat_as_number=prediction_treat_as_number,
|
|||
|
|
prediction_treat_as_bool=prediction_treat_as_bool,
|
|||
|
|
prediction_num_decimals=0,
|
|||
|
|
max_retry=120,
|
|||
|
|
sleep_time=60,
|
|||
|
|
verbose=verbose,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
detailed_results_df_by_instruction_dict[instruction] = detailed_results_df
|
|||
|
|
scores = detailed_results_df["accuracy"]
|
|||
|
|
average_score = np.average(scores)
|
|||
|
|
print(f"instruction: {instruction}, score: {average_score}")
|
|||
|
|
filename = eval_utils.instruction_to_filename(instruction)
|
|||
|
|
file_path = os.path.join(result_by_instruction_folder, f"{filename}.csv")
|
|||
|
|
detailed_results_df.to_csv(file_path, index=True, header=True)
|
|||
|
|
print(f"""saving results of "{instruction}" to {file_path}""")
|
|||
|
|
old_instructions_and_scores.append((instruction, average_score, -1))
|
|||
|
|
old_instructions_and_scores_raw.append((instruction, average_score, -1))
|
|||
|
|
instruction_score_dict[instruction] = average_score
|
|||
|
|
|
|||
|
|
# increment the counter on wrong questions
|
|||
|
|
wrong_question_indices_set = set(
|
|||
|
|
list(
|
|||
|
|
detailed_results_df.iloc[
|
|||
|
|
np.where(detailed_results_df.accuracy == 0.0)[0], :
|
|||
|
|
].index
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
for idx in wrong_question_indices_set:
|
|||
|
|
wrong_questions_from_start_counter[idx] += 1
|
|||
|
|
|
|||
|
|
# evolution
|
|||
|
|
for i_step in range(num_search_steps):
|
|||
|
|
print(f"\n================== Step {i_step} =====================")
|
|||
|
|
if not i_step % 10:
|
|||
|
|
print(f"old_instructions_and_scores: {old_instructions_and_scores}")
|
|||
|
|
|
|||
|
|
if optimizer_llm_temperature_schedule == "linear_increase":
|
|||
|
|
optimizer_llm_temperature_curr = (
|
|||
|
|
optimizer_llm_temperature
|
|||
|
|
+ i_step
|
|||
|
|
/ num_search_steps
|
|||
|
|
* (optimizer_llm_temperature_end - optimizer_llm_temperature)
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
optimizer_llm_temperature_curr = optimizer_llm_temperature
|
|||
|
|
print(
|
|||
|
|
f"current optimizer_llm_temperature: {optimizer_llm_temperature_curr}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# generate new instructions
|
|||
|
|
if few_shot_qa_pairs:
|
|||
|
|
if few_shot_selection_criteria == "accumulative_most_frequent":
|
|||
|
|
# select QA pairs that were done wrong the most number of times
|
|||
|
|
most_frequent_wrong_question_indices = [
|
|||
|
|
k
|
|||
|
|
for k, _ in sorted(
|
|||
|
|
wrong_questions_from_start_counter.items(), key=lambda x: -x[1]
|
|||
|
|
)
|
|||
|
|
]
|
|||
|
|
print(
|
|||
|
|
"len(most_frequent_wrong_question_indices):"
|
|||
|
|
f" {len(most_frequent_wrong_question_indices)}"
|
|||
|
|
)
|
|||
|
|
if (
|
|||
|
|
len(most_frequent_wrong_question_indices)
|
|||
|
|
<= num_few_shot_questions_for_instruction_refinement
|
|||
|
|
):
|
|||
|
|
few_shot_index_list = most_frequent_wrong_question_indices.copy()
|
|||
|
|
else:
|
|||
|
|
np.random.seed(i_step)
|
|||
|
|
few_shot_index_list = np.sort(
|
|||
|
|
np.random.choice(
|
|||
|
|
most_frequent_wrong_question_indices,
|
|||
|
|
num_few_shot_questions_for_instruction_refinement,
|
|||
|
|
replace=False,
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
elif few_shot_selection_criteria == "current_most_frequent":
|
|||
|
|
# show exemplars done wrong most often by currently shown instructions
|
|||
|
|
old_instruction_score_threshold_single_step = (
|
|||
|
|
old_instruction_score_threshold if i_step > 0 else 0
|
|||
|
|
)
|
|||
|
|
_, old_instructions_and_scores_in_meta_prompt = (
|
|||
|
|
gen_ins_and_score_pairs_substr(
|
|||
|
|
old_instructions_and_scores=old_instructions_and_scores,
|
|||
|
|
old_instruction_score_threshold=old_instruction_score_threshold_single_step,
|
|||
|
|
max_num_instructions=max_num_instructions,
|
|||
|
|
return_str_only=False,
|
|||
|
|
num_score_buckets=num_score_buckets,
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
wrong_questions_counter_single_step = collections.Counter()
|
|||
|
|
for ins, _, _ in old_instructions_and_scores_in_meta_prompt:
|
|||
|
|
filename = eval_utils.instruction_to_filename(ins)
|
|||
|
|
file_path = os.path.join(
|
|||
|
|
result_by_instruction_folder, f"{filename}.csv"
|
|||
|
|
)
|
|||
|
|
single_ins_df = pd.read_csv(file_path, index_col=0, header=0)
|
|||
|
|
wrong_question_indices_set_single_old_ins = set(
|
|||
|
|
list(
|
|||
|
|
single_ins_df.iloc[
|
|||
|
|
np.where(single_ins_df.accuracy == 0.0)[0], :
|
|||
|
|
].index
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
for idx in wrong_question_indices_set_single_old_ins:
|
|||
|
|
wrong_questions_counter_single_step[idx] += 1
|
|||
|
|
most_occurred_wrong_questions = [
|
|||
|
|
k
|
|||
|
|
for k, v in wrong_questions_counter_single_step.items()
|
|||
|
|
if v == max(wrong_questions_counter_single_step.values())
|
|||
|
|
]
|
|||
|
|
if (
|
|||
|
|
len(most_occurred_wrong_questions)
|
|||
|
|
< num_few_shot_questions_for_instruction_refinement
|
|||
|
|
):
|
|||
|
|
# pylint: disable=cell-var-from-loop
|
|||
|
|
idx_most_to_least = sorted(
|
|||
|
|
wrong_questions_counter_single_step,
|
|||
|
|
key=lambda x: -wrong_questions_counter_single_step[x],
|
|||
|
|
)
|
|||
|
|
few_shot_index_list = idx_most_to_least[
|
|||
|
|
:num_few_shot_questions_for_instruction_refinement
|
|||
|
|
]
|
|||
|
|
else:
|
|||
|
|
few_shot_index_list = np.sort(
|
|||
|
|
np.random.choice(
|
|||
|
|
most_occurred_wrong_questions,
|
|||
|
|
num_few_shot_questions_for_instruction_refinement,
|
|||
|
|
replace=False,
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
elif few_shot_selection_criteria == "constant":
|
|||
|
|
np.random.seed(0)
|
|||
|
|
few_shot_index_list = np.sort(
|
|||
|
|
np.random.choice(
|
|||
|
|
train_index,
|
|||
|
|
num_few_shot_questions_for_instruction_refinement,
|
|||
|
|
replace=False,
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
assert few_shot_selection_criteria == "random"
|
|||
|
|
np.random.seed(i_step)
|
|||
|
|
few_shot_index_list = np.sort(
|
|||
|
|
np.random.choice(
|
|||
|
|
train_index,
|
|||
|
|
num_few_shot_questions_for_instruction_refinement,
|
|||
|
|
replace=False,
|
|||
|
|
)
|
|||
|
|
).tolist()
|
|||
|
|
|
|||
|
|
few_shot_index_list_by_step_dict[i_step] = few_shot_index_list
|
|||
|
|
|
|||
|
|
meta_prompt = gen_meta_prompt(
|
|||
|
|
old_instructions_and_scores=old_instructions_and_scores,
|
|||
|
|
instruction_pos=instruction_pos,
|
|||
|
|
optimizer_llm_name=optimizer_llm_name,
|
|||
|
|
old_instruction_score_threshold=old_instruction_score_threshold,
|
|||
|
|
max_num_instructions=max_num_instructions,
|
|||
|
|
meta_prompt_type=meta_prompt_type,
|
|||
|
|
few_shot_qa_pairs=few_shot_qa_pairs,
|
|||
|
|
include_qa=include_qa,
|
|||
|
|
data=raw_data,
|
|||
|
|
few_shot_index_list=few_shot_index_list,
|
|||
|
|
instructions_before_exemplars=meta_prompt_instructions_before_exemplars,
|
|||
|
|
num_score_buckets=num_score_buckets,
|
|||
|
|
dataset_name=dataset_name,
|
|||
|
|
task_name=task_name,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
else: # no few-shot exemplars in meta-prompt
|
|||
|
|
few_shot_index_list = []
|
|||
|
|
meta_prompt = gen_meta_prompt(
|
|||
|
|
old_instructions_and_scores=old_instructions_and_scores,
|
|||
|
|
instruction_pos=instruction_pos,
|
|||
|
|
optimizer_llm_name=optimizer_llm_name,
|
|||
|
|
old_instruction_score_threshold=old_instruction_score_threshold,
|
|||
|
|
max_num_instructions=max_num_instructions,
|
|||
|
|
meta_prompt_type=meta_prompt_type,
|
|||
|
|
few_shot_qa_pairs=False,
|
|||
|
|
include_qa=include_qa,
|
|||
|
|
instructions_before_exemplars=meta_prompt_instructions_before_exemplars,
|
|||
|
|
num_score_buckets=num_score_buckets,
|
|||
|
|
dataset_name=dataset_name,
|
|||
|
|
task_name=task_name,
|
|||
|
|
)
|
|||
|
|
print(f"\nmeta_prompt: \n\n{meta_prompt}\n")
|
|||
|
|
meta_prompts.append((meta_prompt, i_step))
|
|||
|
|
remaining_num_instructions_to_generate = (
|
|||
|
|
num_generated_instructions_in_each_step
|
|||
|
|
)
|
|||
|
|
generated_instructions_raw = []
|
|||
|
|
while remaining_num_instructions_to_generate > 0:
|
|||
|
|
optimizer_llm_input_text = meta_prompt
|
|||
|
|
# generate instructions
|
|||
|
|
print(f"current temperature: {optimizer_llm_temperature_curr}")
|
|||
|
|
raw_outputs = call_optimizer_server_func(
|
|||
|
|
optimizer_llm_input_text,
|
|||
|
|
temperature=optimizer_llm_temperature_curr,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# Extract the generated instructions from the optimizer LLM output. Only
|
|||
|
|
# keep some samples if the desired number of remaining instructions
|
|||
|
|
# is smaller than the total number of decodes in this step.
|
|||
|
|
if meta_prompt_type == "both_instructions_and_exemplars":
|
|||
|
|
raw_outputs = raw_outputs[:remaining_num_instructions_to_generate]
|
|||
|
|
if optimizer_llm_name.lower() in {"gpt-3.5-turbo", "gpt-4", "local"}:
|
|||
|
|
if instruction_pos == "A_begin":
|
|||
|
|
start_string = "<Start>"
|
|||
|
|
end_string = "</Start>"
|
|||
|
|
else:
|
|||
|
|
start_string = "<INS>+"
|
|||
|
|
end_string = "</INS>"
|
|||
|
|
for raw_output in raw_outputs:
|
|||
|
|
if start_string not in raw_output:
|
|||
|
|
start_index = 0
|
|||
|
|
else:
|
|||
|
|
start_index = raw_output.index(start_string) + len(start_string)
|
|||
|
|
if end_string not in raw_output:
|
|||
|
|
end_index = len(raw_output)
|
|||
|
|
else:
|
|||
|
|
end_index = raw_output.index(end_string)
|
|||
|
|
new_inst = raw_output[start_index:end_index].strip()
|
|||
|
|
generated_instructions_raw.append(new_inst)
|
|||
|
|
else:
|
|||
|
|
assert optimizer_llm_name.lower() == "text-bison"
|
|||
|
|
generated_instructions_raw += [
|
|||
|
|
extract_string_in_square_brackets(string)
|
|||
|
|
for string in raw_outputs
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
remaining_num_instructions_to_generate -= optimizer_llm_dict[
|
|||
|
|
"batch_size"
|
|||
|
|
]
|
|||
|
|
else:
|
|||
|
|
assert meta_prompt_type == "instructions_only"
|
|||
|
|
max_num_instructions_to_keep_in_each_output = 1
|
|||
|
|
for string in raw_outputs:
|
|||
|
|
generated_instructions_raw += parse_tag_content(string)[
|
|||
|
|
:max_num_instructions_to_keep_in_each_output
|
|||
|
|
]
|
|||
|
|
remaining_num_instructions_to_generate -= (
|
|||
|
|
optimizer_llm_dict["batch_size"]
|
|||
|
|
* max_num_instructions_to_keep_in_each_output
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
generated_instructions_raw = list(
|
|||
|
|
map(eval_utils.polish_sentence, generated_instructions_raw)
|
|||
|
|
)
|
|||
|
|
print(f"\ninitially generated instructions: {generated_instructions_raw}\n")
|
|||
|
|
|
|||
|
|
# do not evaluate old instructions again
|
|||
|
|
generated_instructions = [] # the new instructions generated in this step
|
|||
|
|
for ins in generated_instructions_raw:
|
|||
|
|
ins_md5_hashstring = eval_utils.instruction_to_filename(
|
|||
|
|
ins, md5_hashing=True
|
|||
|
|
)
|
|||
|
|
if ins_md5_hashstring not in old_instruction_md5_hashstrings_set:
|
|||
|
|
generated_instructions.append(ins)
|
|||
|
|
old_instruction_md5_hashstrings_set.add(ins_md5_hashstring)
|
|||
|
|
else:
|
|||
|
|
print(f"already evaluated '{ins}' previously")
|
|||
|
|
generated_instructions = list(set(generated_instructions))
|
|||
|
|
|
|||
|
|
to_evaluate_instructions = []
|
|||
|
|
for instruction in generated_instructions:
|
|||
|
|
if len(instruction) > 500:
|
|||
|
|
print(f"Step {i_step}, instruction: {instruction}, too long, skipped")
|
|||
|
|
continue
|
|||
|
|
if dataset_name == "gsm8k" and any(
|
|||
|
|
char.isdigit() for char in instruction
|
|||
|
|
):
|
|||
|
|
print(
|
|||
|
|
f"Step {i_step}, instruction: {instruction}, contains numbers,"
|
|||
|
|
" skipped"
|
|||
|
|
)
|
|||
|
|
continue
|
|||
|
|
if "INS" in instruction:
|
|||
|
|
print(
|
|||
|
|
f"Step {i_step}, instruction: {instruction}, contains 'INS',"
|
|||
|
|
" skipped"
|
|||
|
|
)
|
|||
|
|
continue
|
|||
|
|
to_evaluate_instructions.append(instruction)
|
|||
|
|
print(f"\nto-evaluate generated instructions: {to_evaluate_instructions}\n")
|
|||
|
|
|
|||
|
|
# evaluate new instructions on the few-shot exemplars in meta-prompt
|
|||
|
|
if few_shot_qa_pairs and evaluate_generated_ins_on_few_shot:
|
|||
|
|
print("evaluating GENERATED instructions on few-shot exemplars")
|
|||
|
|
single_step_eval_on_few_shot = dict()
|
|||
|
|
for instruction in to_evaluate_instructions:
|
|||
|
|
if instruction not in prev_saved_instructions:
|
|||
|
|
print(
|
|||
|
|
f"evaluating Step {i_step}, instruction: {instruction} on"
|
|||
|
|
" few-shot exemplars"
|
|||
|
|
)
|
|||
|
|
detailed_results_df = eval_utils.evaluate_single_instruction(
|
|||
|
|
data=raw_data,
|
|||
|
|
instruction=instruction,
|
|||
|
|
eval_index_all=few_shot_index_list,
|
|||
|
|
batch_size=batch_size,
|
|||
|
|
call_server_func=call_scorer_server_func,
|
|||
|
|
dataset_name=dataset_name,
|
|||
|
|
num_servers=num_servers,
|
|||
|
|
extract_final_answer_by_prompting_again=extract_final_answer_by_prompting_again,
|
|||
|
|
include_qa=include_qa,
|
|||
|
|
evaluate_in_parallel=evaluate_in_parallel,
|
|||
|
|
instruction_pos=instruction_pos,
|
|||
|
|
is_multiple_choice=is_multiple_choice,
|
|||
|
|
prediction_treat_as_number=prediction_treat_as_number,
|
|||
|
|
prediction_treat_as_bool=prediction_treat_as_bool,
|
|||
|
|
prediction_num_decimals=0,
|
|||
|
|
max_retry=5,
|
|||
|
|
sleep_time=180,
|
|||
|
|
verbose=verbose,
|
|||
|
|
)
|
|||
|
|
single_step_eval_on_few_shot[instruction] = detailed_results_df
|
|||
|
|
|
|||
|
|
print(
|
|||
|
|
f"Step {i_step}, single_step_eval_on_few_shot:"
|
|||
|
|
f" {single_step_eval_on_few_shot}\n"
|
|||
|
|
)
|
|||
|
|
generated_ins_on_few_shot_results_dict[i_step] = (
|
|||
|
|
single_step_eval_on_few_shot
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# evaluate OLD instructions on the few-shot exemplars in meta-prompt
|
|||
|
|
if few_shot_qa_pairs and evaluate_old_ins_on_few_shot:
|
|||
|
|
print("evaluating OLD instructions on few-shot exemplars")
|
|||
|
|
single_step_eval_on_few_shot = dict()
|
|||
|
|
for instruction, _, _ in old_instructions_and_scores:
|
|||
|
|
print(
|
|||
|
|
f"evaluating Step {i_step}, instruction: {instruction} on few-shot"
|
|||
|
|
" exemplars"
|
|||
|
|
)
|
|||
|
|
detailed_results_df = eval_utils.evaluate_single_instruction(
|
|||
|
|
data=raw_data,
|
|||
|
|
instruction=instruction,
|
|||
|
|
eval_index_all=few_shot_index_list,
|
|||
|
|
batch_size=scorer_llm_dict["batch_size"],
|
|||
|
|
call_server_func=call_scorer_server_func,
|
|||
|
|
dataset_name=dataset_name,
|
|||
|
|
num_servers=scorer_llm_dict["num_servers"],
|
|||
|
|
extract_final_answer_by_prompting_again=extract_final_answer_by_prompting_again,
|
|||
|
|
include_qa=include_qa,
|
|||
|
|
evaluate_in_parallel=evaluate_in_parallel,
|
|||
|
|
instruction_pos=instruction_pos,
|
|||
|
|
is_multiple_choice=is_multiple_choice,
|
|||
|
|
prediction_treat_as_number=prediction_treat_as_number,
|
|||
|
|
prediction_treat_as_bool=prediction_treat_as_bool,
|
|||
|
|
prediction_num_decimals=0,
|
|||
|
|
max_retry=5,
|
|||
|
|
sleep_time=180,
|
|||
|
|
verbose=verbose,
|
|||
|
|
)
|
|||
|
|
single_step_eval_on_few_shot[instruction] = detailed_results_df
|
|||
|
|
|
|||
|
|
print(
|
|||
|
|
f"Step {i_step}, single_step_eval_on_few_shot:"
|
|||
|
|
f" {single_step_eval_on_few_shot}\n"
|
|||
|
|
)
|
|||
|
|
old_ins_on_few_shot_results_dict[i_step] = single_step_eval_on_few_shot
|
|||
|
|
|
|||
|
|
# evaluate newly generated instructions on the training set
|
|||
|
|
for instruction in to_evaluate_instructions:
|
|||
|
|
if instruction not in prev_saved_instructions:
|
|||
|
|
print(f"""computing the score of "{instruction}" by prompting""")
|
|||
|
|
detailed_results_df = eval_utils.evaluate_single_instruction(
|
|||
|
|
data=raw_data,
|
|||
|
|
instruction=instruction,
|
|||
|
|
eval_index_all=train_index,
|
|||
|
|
batch_size=batch_size,
|
|||
|
|
call_server_func=call_scorer_server_func,
|
|||
|
|
dataset_name=dataset_name,
|
|||
|
|
num_servers=num_servers,
|
|||
|
|
extract_final_answer_by_prompting_again=extract_final_answer_by_prompting_again,
|
|||
|
|
include_qa=include_qa,
|
|||
|
|
evaluate_in_parallel=evaluate_in_parallel,
|
|||
|
|
instruction_pos=instruction_pos,
|
|||
|
|
is_multiple_choice=is_multiple_choice,
|
|||
|
|
prediction_treat_as_number=prediction_treat_as_number,
|
|||
|
|
prediction_treat_as_bool=prediction_treat_as_bool,
|
|||
|
|
prediction_num_decimals=0,
|
|||
|
|
max_retry=5,
|
|||
|
|
sleep_time=180,
|
|||
|
|
verbose=verbose,
|
|||
|
|
)
|
|||
|
|
prev_saved_instructions.add(instruction)
|
|||
|
|
else:
|
|||
|
|
# do not re-evaluate instructions that had been evaluated previously
|
|||
|
|
detailed_results_df = pd.read_csv(
|
|||
|
|
os.path.join(result_by_instruction_folder, f"{instruction}.csv"),
|
|||
|
|
index_col=0,
|
|||
|
|
header=0,
|
|||
|
|
)
|
|||
|
|
print(f"""reading previously saved "{instruction}" information""")
|
|||
|
|
|
|||
|
|
scores = detailed_results_df["accuracy"]
|
|||
|
|
average_score = np.average(scores)
|
|||
|
|
print(
|
|||
|
|
f"Step {i_step}, instruction: {instruction}, score: {average_score}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# increment the counter on wrong questions
|
|||
|
|
wrong_question_indices_set = set(
|
|||
|
|
list(
|
|||
|
|
detailed_results_df[detailed_results_df["accuracy"] == 0.0].index
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
for idx in wrong_question_indices_set:
|
|||
|
|
wrong_questions_from_start_counter[idx] += 1
|
|||
|
|
|
|||
|
|
filename = eval_utils.instruction_to_filename(instruction)
|
|||
|
|
file_path = os.path.join(
|
|||
|
|
result_by_instruction_folder, f"""{filename}.csv"""
|
|||
|
|
)
|
|||
|
|
detailed_results_df.to_csv(file_path, index=True, header=True)
|
|||
|
|
print(f"saving results to {file_path}")
|
|||
|
|
|
|||
|
|
detailed_results_df_by_instruction_dict[instruction] = detailed_results_df
|
|||
|
|
old_instructions_and_scores.append((instruction, average_score, i_step))
|
|||
|
|
instruction_score_dict[instruction] = average_score
|
|||
|
|
|
|||
|
|
# record all generated instructions
|
|||
|
|
for instruction in generated_instructions_raw:
|
|||
|
|
if instruction in instruction_score_dict:
|
|||
|
|
average_score = instruction_score_dict[instruction]
|
|||
|
|
else:
|
|||
|
|
average_score = np.nan
|
|||
|
|
old_instructions_and_scores_raw.append(
|
|||
|
|
(instruction, average_score, i_step)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# =============================== eval ====================================
|
|||
|
|
# every eval_interval steps, evaluate the instructions that were generated
|
|||
|
|
# in the current step and were not skipped
|
|||
|
|
if not i_step % eval_interval:
|
|||
|
|
for instruction in generated_instructions_raw:
|
|||
|
|
# if the instruction wasn't skipped in any step
|
|||
|
|
if instruction in instruction_score_dict:
|
|||
|
|
if instruction not in instruction_eval_score_dict:
|
|||
|
|
detailed_results_df = eval_utils.evaluate_single_instruction(
|
|||
|
|
data=raw_data,
|
|||
|
|
instruction=instruction,
|
|||
|
|
eval_index_all=eval_index,
|
|||
|
|
batch_size=batch_size,
|
|||
|
|
call_server_func=call_scorer_server_func,
|
|||
|
|
dataset_name=dataset_name,
|
|||
|
|
num_servers=num_servers,
|
|||
|
|
extract_final_answer_by_prompting_again=extract_final_answer_by_prompting_again,
|
|||
|
|
include_qa=include_qa,
|
|||
|
|
evaluate_in_parallel=evaluate_in_parallel,
|
|||
|
|
instruction_pos=instruction_pos,
|
|||
|
|
is_multiple_choice=is_multiple_choice_eval,
|
|||
|
|
prediction_treat_as_number=prediction_treat_as_number,
|
|||
|
|
prediction_treat_as_bool=prediction_treat_as_bool,
|
|||
|
|
prediction_num_decimals=0,
|
|||
|
|
max_retry=5,
|
|||
|
|
sleep_time=180,
|
|||
|
|
verbose=verbose,
|
|||
|
|
)
|
|||
|
|
eval_score = np.average(detailed_results_df["accuracy"])
|
|||
|
|
eval_detailed_results_df_dict[instruction] = detailed_results_df
|
|||
|
|
instruction_eval_score_dict[instruction] = eval_score
|
|||
|
|
else:
|
|||
|
|
eval_score = instruction_eval_score_dict[instruction]
|
|||
|
|
print(
|
|||
|
|
f"EVAL: \nStep {i_step}, instruction: {instruction}, eval score:"
|
|||
|
|
f" {eval_score:.2f}"
|
|||
|
|
)
|
|||
|
|
eval_results.append((i_step, instruction, eval_score))
|
|||
|
|
|
|||
|
|
# ===================== save up-to-date results ===========================
|
|||
|
|
results_dict = dict()
|
|||
|
|
results_dict["meta_prompts"] = meta_prompts
|
|||
|
|
results_dict["old_instructions_and_scores"] = list(
|
|||
|
|
old_instructions_and_scores
|
|||
|
|
)
|
|||
|
|
results_dict["old_instructions_and_scores_raw"] = list(
|
|||
|
|
old_instructions_and_scores_raw
|
|||
|
|
)
|
|||
|
|
results_dict["generated_ins_on_few_shot_results_dict"] = (
|
|||
|
|
generated_ins_on_few_shot_results_dict
|
|||
|
|
)
|
|||
|
|
results_dict["old_ins_on_few_shot_results_dict"] = (
|
|||
|
|
old_ins_on_few_shot_results_dict
|
|||
|
|
)
|
|||
|
|
results_dict["few_shot_index_list_by_step_dict"] = (
|
|||
|
|
few_shot_index_list_by_step_dict
|
|||
|
|
)
|
|||
|
|
results_dict["eval_results"] = eval_results
|
|||
|
|
results_dict["eval_detailed_results_df_dict"] = (
|
|||
|
|
eval_detailed_results_df_dict
|
|||
|
|
)
|
|||
|
|
with open(os.path.join(save_folder, "results_dict.pkl"), "wb") as fp:
|
|||
|
|
pickle.dump(results_dict, fp)
|
|||
|
|
print(f"\nsaved all results to\n{save_folder}")
|