大语言模型评估指标BLEU,ROUGE,PPL_ppl大语言模型

# 第一步安装nltk的包-->pip install nltk
from nltk.translate.bleu_score import sentence_bleu
 
def cumulative_bleu(reference, candidate):
 
    bleu_1_gram = sentence_bleu(reference, candidate, weights=(1, 0, 0, 0))
    bleu_2_gram = sentence_bleu(reference, candidate, weights=(0.5, 0.5, 0, 0))
    bleu_3_gram = sentence_bleu(reference, candidate, weights=(0.33, 0.33, 0.33, 0))
    bleu_4_gram = sentence_bleu(reference, candidate, weights=(0.25, 0.25, 0.25, 0.25))
 
    # print('bleu 1-gram: %f' % bleu_1_gram)
    # print('bleu 2-gram: %f' % bleu_2_gram)
    # print('bleu 3-gram: %f' % bleu_3_gram)
    # print('bleu 4-gram: %f' % bleu_4_gram)
 
    return bleu_1_gram, bleu_2_gram, bleu_3_gram, bleu_4_gram
    # return bleu_1_gram, bleu_2_gram
 
# 生成文本
candidate_text = ["This", "is",  "some",  "generated", "text"]
 
# 参考文本列表
reference_texts = [["This", "is",  "a",  "reference", "text"],
                   ["This", "is",  "another", "reference", "text"]]
 
# 计算 Bleu 指标
c_bleu = cumulative_bleu(reference_texts, candidate_text)
 
# 打印结果
 
print("The Bleu score is:", c_bleu)

# 第一步：安装rouge-->pip install rouge
from rouge import Rouge
 
# 生成文本
generated_text = "This is some generated text."
 
# 参考文本列表
reference_texts = ["This is a reference text.", "This is another generated reference text."]
 
# 计算 ROUGE 指标
rouge = Rouge()
scores = rouge.get_scores(generated_text, reference_texts[1])
print(f'scores-->{scores}')
 
# 打印结果
print("ROUGE-1 precision:", scores[0]["rouge-1"]["p"])
print("ROUGE-1 recall:", scores[0]["rouge-1"]["r"])
print("ROUGE-1 F1 score:", scores[0]["rouge-1"]["f"])
# ROUGE-1 precision: 0.8
# ROUGE-1 recall: 0.6666666666666666
# ROUGE-1 F1 score: 0.7272727223140496

import math
# 定义语料库
sentences = [
['I', 'have', 'a', 'pen'],
['He', 'has', 'a', 'book'],
['She', 'has', 'a', 'cat']
]
# 定义语言模型
unigram = {
'I': 1/12,
'have': 1/12,
'a': 3/12,
'pen': 1/12,
'He': 1/12,
'has': 2/12,
'book': 1/12,
'She': 1/12,
'cat': 1/12
}
# 计算困惑度
perplexity = 0
for sentence in sentences:
    print(sentence)
    sentence_prob = 1
    for word in sentence:
        print(f'word-->{word}')
        sentence_prob *= unigram[word]
        print(sentence_prob)
        print('*'*80)
    print(sentence_prob)
 
    sentence_perplexity = 1/sentence_prob
    print(f'sentence_perplexity--》{sentence_perplexity}')
    perplexity += math.log(sentence_perplexity, 2) #以2为底
    print(f'perplexity-->{perplexity}')
    print('----' * 80)
perplexity = 2 ** (-perplexity/len(sentences))
print('困惑度为：', perplexity)
# 困惑度为： 0.000325