SimPO：用序列平均对数概率作为奖励的简洁对齐

python def simpo_loss(beta, gamma, model, prompt_ids, chosen_ids, rejected_ids): # 计算 chosen 响应的对数概率 chosen_logps = model(prompt_ids, chosen_ids).logits.log_softmax(-1) chosen_token_logps = torch.gather(chosen_logps, 2, chosen_ids.unsqueeze(-1)).squeeze(-1) r_chosen = beta * chosen_token_logps.sum(-1) / chosen_ids.size(-1)

# 计算 rejected 响应的对数概率
rejected_logps = model(prompt_ids, rejected_ids).logits.log_softmax(-1)
rejected_token_logps = torch.gather(rejected_logps, 2, rejected_ids.unsqueeze(-1)).squeeze(-1)
r_rejected = beta * rejected_token_logps.sum(-1) / rejected_ids.size(-1)

# 损失函数
logits = r_chosen - r_rejected - gamma
loss = -torch.nn.functional.logsigmoid(logits).mean()
return loss