python-sklearn实现一个简易的智能问答机器人_爬虫实现聊天对话自动应答

随着AI的发展，各大企业采用智能问答机器人取代了人工客服。智能问答系统实现的方法有很多，本篇文章介绍之前做的一个简易的智能问答机器人。采用的方法是使用朴素贝叶斯模型进行问题分类，模糊匹配查询近似问题。

• 实现步骤

1.1 总体流程设计

    问答系统总体实现步骤如下流程图主要包括数据预处理，模型训练，结果映射以及答案匹配。数据预处理主要是对语料库进行收集，并对语料库进行筛选转换为需要的格式；然后使用朴素贝叶斯对处理好的语料库进行训练，语料库输入模型训练之前需要先对其进行预处理以及转换（去停用词，分词，TF-IDF计算），通过TF-IDF计算后则输入朴素贝叶斯中进行训练，由于我的语料库比较简易，所以采用默认参数训练即可达到较好的分类效果。在结果映射步骤中，主要是对事先确定好的类别进行映射处理（可用于脚本网页跳转使用）。答案匹配采用了模糊匹配的方法对用户提的问题进行匹配，搜索出相似的问题并给出其对应的答案。

 

 

1.2 语料库收集

语料库收集如下图。这里第一列为需要分类的类别，第二列为相关的问题。本篇中的语料库主要分为人社信息语料库以及娱乐聊天语料库。

人社信息语料库：

 

娱乐聊天语料库：

 

1.3 主要程序介绍

可视化界面GUI主要采用了tkinter工具包完成，rum_main.py程序如下：

#!/usr/bin/env python3
# _*_ coding:utf-8 _*_
 
from tkinter import *
import time
from speech_test import *
 
'''
定义消息发送函数：
1、在<消息列表分区>的文本控件中实时添加时间；
2、获取<发送消息分区>的文本内容，添加到列表分区的文本中；
3、将<发送消息分区>的文本内容清空。
'''
 
def msgsend():
    msg = '我:' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + '\n'
    # print(msg)
    txt_msglist.insert(END, msg, 'green')  # 添加时间
    query = txt_msgsend.get('0.0', END) #!!!!!!!!!!!!!!!11
    print(query)
    result = main(query)  #问题输入模型入口
    print('result:',result)
    txt_msglist.insert(END, txt_msgsend.get('0.0', END))  # 获取发送消息，添加文本到消息列表
    txt_msglist.insert(END, '\n')
    txt_msgsend.delete('0.0', END)  # 清空发送消息
    robot = '小Y:' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + '\n'
    txt_msglist.insert(END, robot, 'red')
    txt_msglist.insert(END, result+'\n')
 
'''定义取消发送 消息 函数'''
def cancel():
    txt_msgsend.delete('0.0', END)  # 取消发送消息，即清空发送消息
 
'''绑定up键'''
def msgsendEvent(event):
    if event.keysym == 'Up':
        msgsend()
 
tk = Tk()
tk.title('聊天窗口')
 
'''创建分区'''
f_msglist = Frame(height=300, width=300)  # 创建<消息列表分区 >
f_msgsend = Frame(height=300, width=300)  # 创建<发送消息分区 >
f_floor = Frame(height=100, width=300)  # 创建<按钮分区>
f_right = Frame(height=700, width=100)  # 创建<图片分区>
'''创建控件'''
txt_msglist = Text(f_msglist)  # 消息列表分区中创建文本控件
txt_msglist.tag_config('green', foreground='blue')  # 消息列表分区中创建标签
txt_msglist.tag_config('red', foreground='red')  # 消息列表分区中创建标签
txt_msgsend = Text(f_msgsend)  # 发送消息分区中创建文本控件
 
txt_show = Text(f_msglist)  # 消息列表分区中创建文本控件
txt_show.tag_config('red', foreground='red')  # 消息列表分区中创建标签
txt_showsend = Text(f_msgsend)  # 发送消息分区中创建文本控件
 
txt_msgsend.bind('<KeyPress-Up>', msgsendEvent)  # 发送消息分区中，绑定‘UP’键与消息发送。
'''txt_right = Text(f_right) #图片显示分区创建文本控件'''
button_send = Button(f_floor, text='Send',command=msgsend)  # 按钮分区中创建按钮并绑定发送消息函数
button_cancel = Button(f_floor, text='Cancel', command=cancel)  # 分区中创建取消按钮并绑定取消函数
'''分区布局'''
f_msglist.grid(row=0, column=0)  # 消息列表分区
f_msgsend.grid(row=1, column=0)  # 发送消息分区
f_floor.grid(row=2, column=0)  # 按钮分区
f_right.grid(row=0, column=1, rowspan=3)  # 图片显示分区
txt_msglist.grid()  # 消息列表文本控件加载
txt_msgsend.grid()  # 消息发送文本控件加载
button_send.grid(row=0, column=0, sticky=W)  # 发送按钮控件加载
button_cancel.grid(row=0, column=1, sticky=W)  # 取消按钮控件加载
tk.mainloop()
 

智能问答机器人相关程序为 speech_test.py，程序如下：

#-*- coding:utf-8 -*-
import logging
logging.getLogger("requests").setLevel(logging.WARNING)
import csv
import jieba
import pickle
from fuzzywuzzy import fuzz
import math
from scipy import sparse
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from scipy.sparse import lil_matrix
import jieba.posseg as pseg
import sys
import pandas as pd
from sklearn.naive_bayes import MultinomialNB
from speech_recognition import *
import warnings
warnings.filterwarnings("ignore")
 
 
def load_label_url():
    with open('znwd_label_url.csv','r',encoding='utf-8') as f:
        name_id = {}
        label_url = csv.reader(f)
        header = next(label_url)
        for power_name_id in label_url:
            name_id[power_name_id[0]] = power_name_id[1]
    return name_id
 
 
def load_cut_save(filename,load = False):
    jieba.load_userdict('UserDefined_words.txt')
    corpus = []
    label = []
    with open(filename,'rt',encoding='utf-8') as f:
        data_corpus = csv.reader(f)
        header = next(data_corpus)
        for words in data_corpus:
            word = jieba.cut(words[1])
            doc = []
            for x in word:
                 if x not in stop_words and not x.isdigit():
                     doc.append(x)
            corpus.append(' '.join(doc))
            label.append(words[0])
    if load == True:
        with open('corpus.oj','wb') as f:
            pickle.dump(corpus,f)
        with open('label.oj','wb') as f:
            pickle.dump(label,f)
    return corpus,label
 
def train_model():
 
    with open('corpus.oj','rb') as f_corpus:
        corpus = pickle.load(f_corpus)
 
    with open('label.oj','rb') as f_label:
        label = pickle.load(f_label,encoding='bytes')
 
 
    vectorizer = CountVectorizer(min_df=1)
    transformer = TfidfTransformer()
    tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus))
    words_frequency = vectorizer.fit_transform(corpus)
    word = vectorizer.get_feature_names()
    saved = input_tfidf(vectorizer.vocabulary_,sparse.csc_matrix(words_frequency),len(corpus))
    model = MultinomialNB()
    model.fit(tfidf,label)
 
 
    with open('model.oj','wb') as f_model:
        pickle.dump(model,f_model)
 
    with open('idf.oj','wb') as f_idf:
        pickle.dump(saved,f_idf)
 
    return model,tfidf,label
 
class input_tfidf(object):
    def __init__(self,feature_index,frequency,docs):
        self.feature_index = feature_index
        self.frequency = frequency
        self.docs = docs
        self.len = len(feature_index)
 
    def key_count(self,input_words):
        keys = jieba.cut(input_words)
        count = {}
        for key in keys:
            num = count.get(key, 0)
            count[key] = num + 1
        return count
 
    def getTdidf(self,input_words):
        count = self.key_count(input_words)
        result = lil_matrix((1, self.len))
        frequency = sparse.csc_matrix(self.frequency)
        for x in count:
            word = self.feature_index.get(x)
            if word != None and word>=0:
                word_frequency = frequency.getcol(word)
                feature_docs = word_frequency.sum()
                tfidf = count.get(x) * (math.log((self.docs+1) / (feature_docs+1))+1)
                result[0, word] = tfidf
        return result
 
def model_predict(input_str):
    f = open('idf.oj','rb')
    idf = pickle.load(f)
    f.close()
    f = open('model.oj','rb')
    model = pickle.load(f)
    f.close()
    tfidf = idf.getTdidf(input_str)
    classifiction = (model.predict(tfidf))
    # print(model.predict_proba(tfidf))
    prob = model.predict_proba(tfidf).max()
    name_id = load_label_url()
    if prob >= 0.5:
        answer1 = str(classifiction[0],'utf-8')
    else:
        answer1 = None
    return answer1
 
def similarity(input_questions):
    with open('corpus_1233.oj', 'rb') as f:
        corpus = pickle.load(f,encoding='bytes')
 
    with open('question_1233.oj', 'rb') as f:
        question = pickle.load(f,encoding='bytes')
 
    with open('answer_1233.oj', 'rb') as f:
        answer = pickle.load(f,encoding='bytes')
 
 
    text = {}
    train = []
    answer2 = []
    for key, value in enumerate(corpus):
        similarity = fuzz.ratio(input_questions, value)
        if similarity > 40:
            text[key] = similarity
    if len(text) >= 3:
        train = sorted(text.items(), key=lambda d: d[1], reverse=True)
        # print(u"与您提的疑问相似的问题有\n")
        for i in range(3):
            an = {"question":question[train[i][0]],"answer":answer[train[i][0]]}
            answer2.append(an)
            # print("%d、" % (i + 1), \
            #     " 问题：%s\n" % str(question[train[i][0]],'utf-8'), \
            #     " 答案：%s" % str(answer[train[i][0]],'utf-8'))
    elif len(text) == 2:
        train = sorted(text.items(), key=lambda d: d[1], reverse=True)
        # print("与您提的疑问相似的问题有\n")
        for i in range(2):
            an = {"question":question[train[i][0]],"answer":answer[train[i][0]]}
            answer2.append(an)
            # print("%d、" % (i + 1), \
            #     " 问题：%s\n" % str(question[train[i][0]],'utf-8'), \
            #     " 答案：%s" % str(answer[train[i][0]],'utf-8'))
    elif len(text) == 1:
        an = {"question": question[list(text.keys())[0]], "answer": answer[list(text.keys())[0]]}
        answer2.append(an)
        # print("与您提的疑问相似的问题有：\n", \
        #     " 问题：%s" % str(question[text.keys()[0]],'utf-8'), \
        #     " 答案：%s" % str(answer[text.keys()[0]],'utf-8'))
    else:
        # print("您所提的疑问无其他相似问题！")
        an = {"question":None,"answer":None}
        answer2.append(an)
    return answer2
 
def get_greeting(input_questions,question,answer):
    text = {}
    for key, value in enumerate(question):
        similarity = fuzz.ratio(input_questions, value)
        if similarity > 60:
            text[key] = similarity
    if len(text) > 0:
        train = sorted(text.items(), key=lambda d: d[1], reverse=True)
        answer3 = answer[train[0][0]]
    else:
        answer3 = None
    return  answer3
 
 
def sim(doc):
    input_questions = ''
    input_words = jieba.cut(doc)
 
    for x in input_words:
        if x not in stop_words:
            input_questions += x
 
    answer2 = similarity(input_questions)
    return answer2
 
def ans_show(returnSet):
    if returnSet[2] is not None:
        ans = "%s"%returnSet[2]
    elif returnSet[0] is not None:
        ans = "您的问题属于<%s>专栏\n"%returnSet[0]
        ans1 = ""
        if returnSet[1][0]['question'] is not None:
            ans1 = "小Y还知道其他一些问题例如：\n"
            ans2 = ""
            for i in range(len(returnSet[1])):
                ans2 = ans2 + "%d、" % (i + 1) + " 问题：%s\n" % str(returnSet[1][i]['question'],'utf-8') + " 答案：%s" % str(returnSet[1][i]['answer'],'utf-8')
            ans1 = ans1 + ans2
        ans = ans + ans1
    elif returnSet[1][0]['question'] is not None:
        ans1 = "小Y知道相似的问题：\n"
        ans2 = ""
        for i in range(len(returnSet[1])):
            ans2 = ans2 + "%d、" % (i + 1) + " 问题：%s\n" % str(returnSet[1][i]['question'], 'utf-8') + " 答案：%s" % str(returnSet[1][i]['answer'], 'utf-8')
        ans = ans1 + ans2
    else:
        ans = "您问的问题太过深奥，Mike才疏学浅暂时无法为您解答，待我读书破万卷后成为您的百科机器人"
    return ans
 
with open('stop_words.txt', 'rb') as f:
    stop_words = f.read().splitlines()
 
question_greeting = []
answer_greeting = []
with open("greeting.csv", 'r',encoding='utf-8') as f:
    greeting = csv.reader(f)
    header = next(greeting)
    for words in greeting:
        question_greeting.append(words[0])
        answer_greeting.append(words[1])
 
filename = 'znwd_corpus.csv'
corpus, label = load_cut_save(filename,load=False)
 
def main(question):
    if question != None:
        query = question    #########此处会因语音无法识别还报错
        print("我 > %s" %query)
       ##############
        answer3 = get_greeting(query,question_greeting,answer_greeting)
        # print(answer3)
        if answer3 is None:
            answer1 = model_predict(query)
            answer2 = sim(query)
        else:
            answer1 = None
            answer2 = None
        ans = [answer1,answer2,answer3]
        result = ans_show(ans)
    else:
        result = "输入有误请重新输入！"
        query = None
    return result
 
 
 

因而，一个简单的智能问答机器人即可实现，若需要问答机器人能够回答更多的内容，可针对语料库进行修改，进而丰富智能问答机器人的聊天范围。程序中读取语料库采用了pickle工具包将预处理后的语料库进行序列化至本地，进而在下次使用语料库不需要再次进行预处理，可节省处理的时间。修改语料库后需重新覆盖序列化至本地的语料库文件和模型文件。

• 效果展示

最终效果如下，我的语料库主要用了社保卡相关的数据，该问答系统可以当作是一个社保信息方面的问答。

 

 

工程源码：https://github.com/liangjunAI/chatting_robot