使用直接偏好优化策略微调Mistral-7b模型

pip install -q datasets trl peft bitsandbytes sentencepiece wandb

import os
import gc
import torch

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig
from datasets import load_dataset
from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training
from trl import DPOTrainer
import bitsandbytes as bnb
from google.colab import userdata
import wandb

# 在Google Colab秘密的标签中定义
hf_token = userdata.get('huggingface')
wb_token = userdata.get('wandb')
wandb.login(key=wb_token)

model_name = "teknium/OpenHermes-2.5-Mistral-7B"
new_model = "NeuralHermes-2.5-Mistral-7B"

<|im_start|>system
You are a helpful chatbot assistant.<|im_end|>
<|im_start|>user
Hi<|im_end|>
<|im_start|>assistant
Hi, how can I help you?<|im_end|>

def chatml_format(example):
 # 格式化系统列
 if len(example['system']) > 0:
 message = {"role": "system", "content": example['system']}
 system = tokenizer.apply_chat_template([message], tokenize=False)
 else:
 system = ""

 # 格式化指令
 message = {"role": "user", "content": example['question']}
 prompt = tokenizer.apply_chat_template([message], tokenize=False, add_generation_prompt=True)

 # 设置所选答案的格式
 chosen = example['chosen'] + "<|im_end|>\n"

 # 格式化拒绝的答案
 rejected = example['rejected'] + "<|im_end|>\n"

 return {
 "prompt": system + prompt,
 "chosen": chosen,
 "rejected": rejected,
 }

# 加载数据集
dataset = load_dataset("Intel/orca_dpo_pairs")['train']

# 保存列数据
original_columns = dataset.column_names

# 分词器
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "left"

# 格式化数据集
dataset = dataset.map(
 chatml_format,
 remove_columns=original_columns
)

{'prompt': '<|im_start|>system\nYou are an AI assistant. You will be given a task. You must generate a detailed and long answer.<|im_end|>\n<|im_start|>user\nGenerate an approximately fifteen-word sentence that describes all this data: Midsummer House eatType restaurant; Midsummer House food Chinese; Midsummer House priceRange moderate; Midsummer House customer rating 3 out of 5; Midsummer House near All Bar One<|im_end|>\n<|im_start|>assistant\n',
'chosen': 'Midsummer House is a moderately priced Chinese restaurant with a 3/5 customer rating, located near All Bar One.<|im_end|>\n',
'rejected': ' Sure! Here\'s a sentence that describes all the data you provided:\n\n"Midsummer House is a moderately priced Chinese restaurant with a customer rating of 3 out of 5, located near All Bar One, offering a variety of delicious dishes."<|im_end|>\n'}

# LoRA配置
peft_config = LoraConfig(
 r=16,
 lora_alpha=16,
 lora_dropout=0.05,
 bias="none",
 task_type="CAUSAL_LM",
 target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj']
)

# 要微调的模型
model = AutoModelForCausalLM.from_pretrained(
 model_name,
 torch_dtype=torch.float16,
 load_in_4bit=True
)
model.config.use_cache = False

# 参考模型
ref_model = AutoModelForCausalLM.from_pretrained(
 model_name,
 torch_dtype=torch.float16,
 load_in_4bit=True
)

# 训练参数
training_args = TrainingArguments(
 per_device_train_batch_size=4,
 gradient_accumulation_steps=4,
 gradient_checkpointing=True,
 learning_rate=5e-5,
 lr_scheduler_type="cosine",
 max_steps=200,
 save_strategy="no",
 logging_steps=1,
 output_dir=new_model,
 optim="paged_adamw_32bit",
 warmup_steps=100,
 bf16=True,
 report_to="wandb",
)

# 创建DPO训练器
dpo_trainer = DPOTrainer(
 model,
 ref_model,
 args=training_args,
 train_dataset=dataset,
 tokenizer=tokenizer,
 peft_cnotallow=peft_config,
 beta=0.1,
 max_prompt_length=1024,
 max_length=1536,
)

# 使用DPO微调模型
dpo_trainer.train()

#保存所有工作
dpo_trainer.model.save_pretrained("final_checkpoint")
tokenizer.save_pretrained("final_checkpoint")

# 刷新内存
del dpo_trainer, model, ref_model
gc.collect()
torch.cuda.empty_cache()

# 在FP16（而不是NF4）中重新加载模型
base_model = AutoModelForCausalLM.from_pretrained(
 model_name,
 return_dict=True,
 torch_dtype=torch.float16,
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

# 将基本模型与适配器合并
model = PeftModel.from_pretrained(base_model, "final_checkpoint")
model = model.merge_and_unload()

# 保存模型和分词器
model.save_pretrained(new_model)
tokenizer.save_pretrained(new_model)

# 然后将其推送到Hugging Face中心
model.push_to_hub(new_model, use_temp_dir=False, token=hf_token)
tokenizer.push_to_hub(new_model, use_temp_dir=False, token=hf_token)

#格式化提示
message = [
 {"role": "system", "content": "You are a helpful assistant chatbot."},
 {"role": "user", "content": "What is a Large Language Model?"}
]
tokenizer = AutoTokenizer.from_pretrained(new_model)
prompt = tokenizer.apply_chat_template(message, add_generation_prompt=True, tokenize=False)

# 创建管道
pipeline = transformers.pipeline(
 "text-generation",
 model=new_model,
 tokenizer=tokenizer
)

# 生成文本
sequences = pipeline(
 prompt,
 do_sample=True,
 temperature=0.7,
 top_p=0.9,
 num_return_sequences=1,
 max_length=200,
)
print(sequences[0]['generated_text'])

A large language model is a type of artificial intelligence (AI) system that has been trained on vast amounts of text data. These models are designed to understand and generate human language, allowing them to perform various natural language processing tasks, such as text generation, language translation, and question answering. Large language models typically use deep learning techniques, like recurrent neural networks (RNNs) or transformers, to learn patterns and relationships in the data, enabling them to generate coherent and contextually relevant responses. The size of these models, in terms of the number of parameters and the volume of data they are trained on, plays a significant role in their ability to comprehend and produce complex language structures.