selenium+webdriver 案例实战_selenium.webdriver 案例

一、环境准备

1. 代码环境：python3.6
2. chrome环境：在chrome浏览器中敲入chrome://version。前往链接下载对应版本的chormedriver.exe。
   
3. 所需第三方库：Beautifulsoup4、selenium、time、pandas（读写csv）

二、咨询师信息爬取

网址：网址
目标：爬取共205咨询师的ID和姓名。
分析：姓名就不多解释，主要解释ID。打开开发者工具，随便定位一个咨询师。可以定位其在html中的标签。我们只需要解析出href的数据，抽取其中的数字即可。

由于采集咨询师的ID和姓名并不需要点击进入咨询师的详情界面，因此这里不需要使用webdriver，可以使用bs4等方式解析网页获取数据，这样速度更快，编写代码也更简单！完整代码如下：

#!/usr/bin/env python
# encoding=utf-8
"""
    Author：YJY
    Need:爬取壹点零网站的咨询师列表LIST
"""

import codecs
import requests
from bs4 import BeautifulSoup
import pandas as pd
import warnings
warnings.filterwarnings('ignore')


DOWNLOAD_URL = 'https://ydl.com/experts/'


def download_page(url):
    return requests.get(url, headers={
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
    }).content


def parse_html(html, list):
    # print("parse_html执行一次")
    try:
        soup = BeautifulSoup(html)
        doctor_list_soup = soup.find('div', attrs={'class': 'expertsList_items'})
        # find_all获取该页面的所有咨询师信息。每个咨询师完整信息包含在<item>标签下。
        for doctor_item in doctor_list_soup.find_all('div', attrs={'class': 'item'}):
            doctor_info = doctor_item.find('div', attrs={'class': 'info'})
            doctor_information = doctor_info.find('h3').find('a')
            # 获取姓名
            doctor_name = doctor_information.getText()
            # 获取id
            doctor_id = doctor_information['href']
            doctor_id = ''.join(filter(lambda i: i in ['#'] or i.isdigit(), doctor_id))
            # 将姓名和ID添加至咨询师信息列表
            doctor_information_list.append([doctor_id, doctor_name.replace('\n','').replace(' ','')])

        # 判断下一页是否存在
        next_page = soup.find('li', attrs={'class': 'next'}).find('a')
        print(next_page['href'])
        if next_page:
        	#下一页存在，则返回当前的咨询师列表信息和下一页的url
            return doctor_information_list, DOWNLOAD_URL + next_page['href']
        # 下一页不存在，则返回当前的咨询师列表，url设置为None终止while循环
        return doctor_information_list, None
	# 对采集的异常进行简单处理
    except Exception as e:
        return doctor_information_list, DOWNLOAD_URL + next_page['href']

def main():
    url = DOWNLOAD_URL
    page = 0
    doctors = []
    try:
        while url:
            # 打印当前正在打印的页数(前两次的抓取结果均为第一页，没有太仔细想这个bug，不过问题不大)
            print("正在打印第%d页" % page)
            page += 1
            # 解析函数
            try:
                html = download_page(url)
                doctors, url = parse_html(html, doctors)
            except Exception as e:
            	# 第102页会异常（具体原因没有细想），所以遇到异常时手动设置下一页URL
                html = download_page(DOWNLOAD_URL + '/experts?experts=&page=' + str(page))
    # 简单处理异常
    except Exception as e:
        print(e)
    # 将列表转换为DataFrame，导出至csv文件
    name = ['DocID', 'DocName']
    doctor_information = pd.DataFrame(columns=name, data=doctors)
    doctor_information.to_csv('data/doctor_information.csv', encoding='utf-8', index=None)


if __name__ == '__main__':
    main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
'
运行
运行

这样我们就得到了所有咨询师的ID和姓名信息，便于后续数据的采集。代码写的比较匆忙，还可以有很多修改，仅仅为了实现功能所以糙了点，异常处理都很随意哈哈哈哈，不要太纠结代码。

三、问答数据的采集

下面开始代码编写：

放在前面，对于xpath和selenium、webdriver不熟悉的朋友参考以下教程：

• xpath：xpath教程
• webdriver：selenium+webdriver教程

首先，导入相关包

import pandas as pd
import time
from selenium import webdriver
1
2
3
'
运行
运行

设置YiDianLing类，包含五个属性和两个函数。

class YiDianLing(object):

    def __init__(self):
        self.url = "https://ydl.com/experts/"
        # 创建浏览器
        self.driver = webdriver.Chrome(r"chromedriver.exe")
        # 创建子浏览器（因为webdriver返回前一页，可能会发生数据丢失，因此我们直接采用子浏览器打开咨询师的具体问答，主浏览器仍保持咨询师问答列表，这里有很多坑，大家需要小心！）
        self.driver_son = webdriver.Chrome(r"chromedriver.exe")
        # 读取咨询师的ID和姓名
        self.DoctorIdlist = pd.read_csv("data/doctor_information.csv")['DocID']
        self.DoctorNamelist = pd.read_csv("data/doctor_information.csv")['DocName']

	# 爬取问答数据
    def AskDataCrawl(self):
 	# 爬取文章数据（不多说明，代码在最后）
    def ArticleDataCrawl(self):
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

创建问答数据列表（最后将该列表转换为DataFrame，导出csv）。遍历前面爬取的咨询师信息文件，遍历每个咨询师的主页。

# 创建问答数据列表
ArticleData = []
# 遍历咨询师列表
for i in range(len(self.DoctorIdlist)):
    # 访问第i+1个咨询师的主界面
        doctorurl = self.url + str(self.DoctorIdlist[i])
        self.driver.get(doctorurl)
        time.sleep(1)
1
2
3
4
5
6
7
8

点击页面中的“查看所有”，跳转到问答界面。

# 跳转到问答板快（查看所有）
self.driver.find_element_by_link_text('查看全部').click()
# 关闭旧界面
self.driver.switch_to.window(self.driver.window_handles[0])
self.driver.close()
# 设置当前页面为最后一个界面
self.driver.switch_to.window(self.driver.window_handles[-1])
time.sleep(1)
1
2
3
4
5
6
7
8

遍历当前界面的所有问答数据

# 获取所有问答
questionsList = self.driver.find_elements_by_class_name('item')
for question in questionsList:
	time.sleep(3)
	try:# 获取问题URL
		QuestionUrl = question.find_element_by_tag_name('a').get_attribute('href')
        print(QuestionUrl)
        # 获取问题ID和咨询师回复ID
        QuestionID = QuestionUrl.split('/')[4]
        AnswerID = QuestionID + str(self.DoctorIdlist[i])
        # 打开子浏览器
        self.driver_son.get(QuestionUrl)
        time.sleep(1)
        # 获取问题内容
        QuestionText = self.driver_son.find_element_by_xpath('//*[@class="content"]/p').text
        QuestionText = QuestionText.replace('\n', '')
        # 获取提问时间
        QuestionDate = self.driver_son.find_element_by_class_name('ask_right').text.split('\n')
        QuestionDate = QuestionDate[0]
        # 咨询师回复时间
        AnswerDate = self.driver_son.find_element_by_xpath('//*[@class="text first-lever"]/time').text
        # 获取咨询师回复内容
        AnswerText = self.driver_son.find_element_by_xpath('//*[@class="text first-lever"]/p').text
        AnswerText = AnswerText.replace('\n', '')
        # 获取咨询师回复收到的感谢数量
        AnswerThanks = self.driver_son.find_element_by_xpath('//*[@class="votable"]/a/font').text
        # 添加至列表
        AskData.append([self.DoctorIdlist[i], self.DoctorNamelist[i], QuestionID, QuestionText, QuestionDate, AnswerID, AnswerDate, AnswerText, AnswerThanks])
        
	except Exception as e:
		continue
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

跳转至下一页，直至无法跳转时退出循环，遍历下一个咨询师的主页，重复上述循环。

# 跳转到下一页
try:
	page += 1
    self.driver.get('https://ydl.com/experts/' + str(self.DoctorIdlist[i]) + '/answerList/p' + str(page))
    time.sleep(3)
except Exception as e:
    break
1
2
3
4
5
6
7

完整代码如下，包括问大数据获取和文章数据的获取。代码编写比较匆忙，或多或少有些小问题，还请大家见谅！

#!/usr/bin/env python
# encoding=utf-8
"""
    Author：YJY
    Need:爬取壹点零网站的咨询师问答数据和文章数据
"""

import pandas as pd
import time
from selenium import webdriver


class YiDianLing(object):

    def __init__(self):
        self.url = "https://ydl.com/experts/"
        # 创建浏览器
        self.driver = webdriver.Chrome(r"chromedriver.exe")
        self.driver_son = webdriver.Chrome(r"chromedriver.exe")
        self.DoctorIdlist = pd.read_csv("data/doctor_information.csv")['DocID']
        self.DoctorNamelist = pd.read_csv("data/doctor_information.csv")['DocName']

    def AskDataCrawl(self):
        # 创建问答数据列表
        AskData = []
        # 遍历咨询师列表
        nums = 1
        for i in range(len(self.DoctorIdlist)):
            # 访问第i个咨询师的主界面
            doctorurl = self.url + str(self.DoctorIdlist[i])
            self.driver.get(doctorurl)
            time.sleep(1)
            # 跳转到问答板快（查看所有）
            self.driver.find_element_by_link_text('查看全部').click()
            self.driver.switch_to.window(self.driver.window_handles[0])
            self.driver.close()
            self.driver.switch_to.window(self.driver.window_handles[-1])
            time.sleep(1)
            # 问题列表界面page
            page = 1
            while True:
                # 获取所有问答
                questionsList = self.driver.find_elements_by_class_name('item')
                for question in questionsList:
                    time.sleep(3)
                    try:
                        # 获取问题URL
                        QuestionUrl = question.find_element_by_tag_name('a').get_attribute('href')
                        print(QuestionUrl)
                        # 获取问题ID和咨询师回复ID
                        QuestionID = QuestionUrl.split('/')[4]
                        AnswerID = QuestionID + str(self.DoctorIdlist[i])
                        # 打开子浏览器
                        self.driver_son.get(QuestionUrl)
                        time.sleep(1)

                        # 获取问题内容
                        QuestionText = self.driver_son.find_element_by_xpath('//*[@class="content"]/p').text
                        QuestionText = QuestionText.replace('\n', '')
                        # 获取提问时间
                        QuestionDate = self.driver_son.find_element_by_class_name('ask_right').text.split('\n')
                        QuestionDate = QuestionDate[0]
                        # 咨询师回复时间
                        AnswerDate = self.driver_son.find_element_by_xpath('//*[@class="text first-lever"]/time').text
                        # 获取咨询师回复内容
                        AnswerText = self.driver_son.find_element_by_xpath('//*[@class="text first-lever"]/p').text
                        AnswerText = AnswerText.replace('\n', '')
                        # 获取咨询师回复收到的感谢数量
                        AnswerThanks = self.driver_son.find_element_by_xpath('//*[@class="votable"]/a/font').text
                        # 添加至列表
                        AskData.append([self.DoctorIdlist[i], self.DoctorNamelist[i], QuestionID, QuestionText, QuestionDate, AnswerID, AnswerDate, AnswerText, AnswerThanks])
                        print(AskData)
                        # self.driver_son.close()
                    except Exception as e:
                        continue

                # 跳转到下一页
                try:
                    page += 1
                    self.driver.get('https://ydl.com/experts/' + str(self.DoctorIdlist[i]) + '/answerList/p' + str(page))
                    time.sleep(3)
                except Exception as e:
                    break
            nums += 1
        # 导出csv
        Name = ['DocID', 'DocName', 'QuestionID', 'QuestionText', 'QuestionDate', 'AnswerID', 'AnswerDate', 'AnswerText', 'AnswerThanks']
        df = pd.DataFrame(columns=Name, data=AskData)
        df.to_csv('data/doctor_ask.csv')


    def ArticleDataCrawl(self):
        # 创建问答数据列表
        ArticleData = []
        # 遍历咨询师列表
        for i in range(len(self.DoctorIdlist)):
            # 访问第i+1个咨询师的主界面
            doctorurl = self.url + str(self.DoctorIdlist[i])
            self.driver.get(doctorurl)
            time.sleep(1)
            # 判断是否有文章，有则进入该板块
            try:
                if '文章' in self.driver.find_element_by_xpath('//*[@class="content-nav"]/li[5]').text:
                    self.driver.find_element_by_xpath('//*[@class="content-nav"]/li[5]').click()
                    self.driver.switch_to.window(self.driver.window_handles[0])
                    self.driver.close()
                    self.driver.switch_to.window(self.driver.window_handles[-1])
                    time.sleep(1)
                elif '文章' in self.driver.find_element_by_xpath('//*[@class="content-nav"]/li[6]').text:
                    self.driver.find_element_by_xpath('//*[@class="content-nav"]/li[6]').click()
                    self.driver.switch_to.window(self.driver.window_handles[0])
                    self.driver.close()
                    self.driver.switch_to.window(self.driver.window_handles[-1])
                    time.sleep(1)
                else:
                    continue
            except Exception as e:
                continue

            # 文章列表界面page
            page = 1
            err_appear = False
            while not err_appear:
                articlesList = self.driver.find_elements_by_xpath('//*[@class="ui-content testings index testings_index chrome modern mac webkit desktop"]/div')
                for article in articlesList:
                    time.sleep(3)
                    try:
                        # 获取文章URL
                        ArticleUrl = article.find_element_by_tag_name('a').get_attribute('href')
                        print(ArticleUrl)
                    except:
                        err_appear = True
                        break
                    # 获取文章ID
                    ArticleID = ArticleUrl.split('/')[4]
                    # 打开子浏览器
                    self.driver_son.get(ArticleUrl)
                    time.sleep(1)
                    try:
                        # 获取文章发表时间和文章阅读量和获赞数
                        ArticleInfo = self.driver_son.find_element_by_xpath('//*[@class="post_desc"]').text
                        ArticleDate = ArticleInfo.split('   ')[1]
                        ArticleViews = ArticleInfo.split('   ')[2]
                        ArticleViews = ''.join(filter(str.isdigit, ArticleViews))
                        ArticleVotes = ArticleInfo.split('   ')[3]
                        ArticleVotes = ''.join(filter(str.isdigit, ArticleVotes))
                        # 获取文章内容
                        ArticleText = self.driver_son.find_element_by_class_name('event_content').text
                        ArticleText = ArticleText.replace('\n', '')
                        # 获取文章图片数量
                        ArticleImageNum = len(self.driver_son.find_elements_by_xpath('//*[@class="event_content"]//img'))
                        # 添加至列表
                        ArticleData.append([self.DoctorIdlist[i], self.DoctorNamelist[i], ArticleID, ArticleDate, ArticleImageNum, ArticleText, ArticleViews, ArticleVotes])
                        print(ArticleData)

                    except Exception as e:
                        continue

                # 跳转到下一页
                try:
                    page += 1
                    self.driver.get('https://www.ydl.com/experts/' + str(self.DoctorIdlist[i]) + '/jingyan/p' + str(page))
                    time.sleep(3)
                except Exception as e:
                    break
        # 导出csv
        Name = ['DocID', 'DocName', 'ArticleID', 'ArticleDate', 'ArticleImageNum', 'ArticleText', 'ArticleViews', 'ArticleVotes']
        df = pd.DataFrame(columns=Name, data=ArticleData)
        df.to_csv('data/doctor_article.csv')


if __name__ == '__main__':
    yidianling = YiDianLing()
    yidianling.AskDataCrawl()
    yidianling.ArticleDataCrawl()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174