devbox
不正经
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

Python+selenium 爬取贝壳房源数据_爬虫获取贝壳二手房数据

作者：不正经 | 2024-03-08 21:42:47

赞

踩

爬虫获取贝壳二手房数据

提示：点开主页可以下载全部代码跟数据进行参考！

文章目录

前言

本文仅用于技术分享交流，如有用于其他用途的，自行承担后果

一、封装selenium

  1. from selenium import webdriver
  2. # from selenium.webdriver.chrome.options import Options
  3. import undetected_chromedriver as uc
  4.  
  5. def share_browser():
  6.     # 初始化
  7.     chrome_options = uc.ChromeOptions()
  8.     # chrome_options.add_argument((f'--proxy-server=http://27.150.162.104:4278'))
  9.     # chrome_options.add_argument('--headless')
  10.     chrome_options.add_argument('--disable-gpu')
  11.     chrome_options.add_argument( 'service_args = [’–ignore - ssl - errors = true’, ‘–ssl - protocol = TLSv1’]')  # 忽略ssl验证
  12.     chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
  13.     chrome_options.add_argument('--no-sandbox')
  14.     chrome_options.add_argument("disable-infobars")
  15.     # options.add_argument('--start-maximized')
  16.     # options.add_argument('--start-fullscreen')
  17.     chrome_options.add_argument('--single-process')
  18.     chrome_options.add_argument('--disable-dev-shm-usage')
  19.     chrome_options.add_argument('--disable-blink-features=AutomationControlled')
  20.     chrome_options.add_experimental_option('useAutomationExtension', False)
  21.     chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
  22.     chrome_options.add_argument("log-level=3")
  23.  
  24.     chrome_options.add_argument("--disable-extensions")
  25.     chrome_options.add_argument("--disable-popup-blocking")
  26.     chrome_options.add_argument("--profile-directory=Default")
  27.     chrome_options.add_argument("--ignore-certificate-errors")
  28.     chrome_options.add_argument("--disable-plugins-discovery")
  29.     chrome_options.add_argument("--incognito")
  30.     chrome_options.add_argument('--no-first-run')
  31.     chrome_options.add_argument('--no-service-autorun')
  32.     chrome_options.add_argument('--no-default-browser-check')
  33.     chrome_options.add_argument('--password-store=basic')
  34.  
  35.     chrome_options.add_argument(
  36.         'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36')
  37.  
  38.     # 这个路径是谷歌浏览器的路径
  39.     path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
  40.     chrome_options.binary_location = path
  41.  
  42.     browser = webdriver.Chrome(chrome_options=chrome_options)
  43.     browser.maximize_window()
  44.  
  45.     return browser

二、抓取流程

1.翻页功能

代码如下（示例）：

  1. def get_href(page):
  2.     print(f'获取第:{page}页数据...')
  3.     url = f'https://bj.zu.ke.com/zufang/pg{page}rs%E5%85%AC%E5%AF%93/#contentList'
  4.     res = requests.get(url)
  5.     tree = etree.HTML(res.text)
  6.     href_list = tree.xpath('//*[@id="content"]/div[1]/div[1]/div/div/p[1]/a/@href')
  7.     return href_list

2.通过xpath获取数据

代码如下（示例）：

  1. def get_content(href):
  2.     url = 'https://bj.zu.ke.com/' + href
  3.     with open('used_url.txt', 'r', encoding='utf-8') as fp:
  4.         used_url_list = [f.strip('\n') for f in fp.readlines()]
  5.     data_list = []
  6.     if url not in used_url_list:
  7.         try:
  8.             driver.get(url)
  9.  
  10.             # wait.until(EC.invisibility_of_element_located((By.XPATH, '/html/body/div[3]/div[1]/div[10]/div/div[1]/img')))
  11.             # driver.find_element(By.XPATH, '/html/body/div[3]/div[1]/div[10]/div/div[1]/img').click()
  12.             tree = etree.HTML(driver.page_source)
  13.             # 标题
  14.             title = tree.xpath('/html/body/div[3]/div[1]/div[3]/p/text()')[0].strip('\n').strip(' ')
  15.             # 价格
  16.             price = tree.xpath('//*[@id="aside"]/div[1]/span/text()')  # 元/月
  17.             price = price[0] if price else ''
  18.             # 房屋类型
  19.             house_class = tree.xpath('//*[@id="aside"]/ul/li[2]/text()')[0]
  20.             # 面积
  21.             area = tree.xpath('//*[@id="info"]/ul[1]/li[2]/text()')[0].replace('面积：', ' ')
  22.             # 维护
  23.             maintain = tree.xpath('//*[@id="info"]/ul[1]/li[5]/text()')[0].replace('维护：', ' ')
  24.             # 楼层
  25.             floor = tree.xpath('//*[@id="info"]/ul[1]/li[8]/text()')[0].replace('楼层：', ' ')
  26.             # 车位
  27.             parking_lot = tree.xpath('//*[@id="info"]/ul[1]/li[11]/text()')[0].replace('车位：', ' ')
  28.             # 用电
  29.             use_cable = tree.xpath('//*[@id="info"]/ul[1]/li[14]/text()')[0].replace('用电：', ' ')
  30.             # 采暖
  31.             heating = tree.xpath('//*[@id="info"]/ul[1]/li[17]/text()')[0].replace('采暖：', ' ')
  32.             # 朝向
  33.             point = tree.xpath('//*[@id="info"]/ul[1]/li[3]/text()')[0].replace('朝向：', ' ')
  34.             # 入住时间
  35.             move_into_time = tree.xpath('//*[@id="info"]/ul[1]/li[6]/text()')[0].replace('入住：', ' ')
  36.             # 电梯
  37.             lift = tree.xpath('//*[@id="info"]/ul[1]/li[9]/text()')[0].replace('电梯：', ' ')
  38.             # 用水
  39.             use_water = tree.xpath('//*[@id="info"]/ul[1]/li[12]/text()')[0].replace('用水：', ' ')
  40.             # 燃气
  41.             gas = tree.xpath('//*[@id="info"]/ul[1]/li[15]/text()')[0].replace('燃气：', ' ')
  42.             # 租期
  43.             tenancy = tree.xpath('//*[@id="info"]/ul[2]/li[2]/text()')[0].replace('租期：', ' ')
  44.  
  45.             # 状态
  46.             # 洗衣机
  47.             xiyiji = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[2]/@class')[0]
  48.             status1 = '有' if 'no' not in xiyiji else '无'
  49.             # 空调
  50.             kongtiao = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[3]/@class')[0]
  51.             status2 = '有' if 'no' not in kongtiao else '无'
  52.             # 衣柜
  53.             yigui = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[4]/@class')[0]
  54.             status3 = '有' if 'no' not in yigui else '无'
  55.             # 电视
  56.             dianshi = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[5]/@class')[0]
  57.             status4 = '有' if 'no' not in dianshi else '无'
  58.             # 冰箱
  59.             bingxiang = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[6]/@class')[0]
  60.             status5 = '有' if 'no' not in bingxiang else '无'
  61.             # 热水器
  62.             reshuqi = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[7]/@class')[0]
  63.             status6 = '有' if 'no' not in reshuqi else '无'
  64.             # 床
  65.             chuang = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[8]/@class')[0]
  66.             status7 = '有' if 'no' not in chuang else '无'
  67.             # 暖气
  68.             nuanqi = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[9]/@class')[0]
  69.             status8 = '有' if 'no' not in nuanqi else '无'
  70.             # 宽带
  71.             kuandai = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[10]/@class')[0]
  72.             status9 = '有' if 'no' not in kuandai else '无'
  73.             # 天然气
  74.             tianranqi = tree.xpath('/html/body/div[3]/div[1]/div[3]/div[3]/div[2]/ul/li[11]/@class')[0]
  75.             status10 = '有' if 'no' not in tianranqi else '无'
  76.             # 地铁站距离
  77.             # distance = tree.xpath('//*[@id="around"]/ul[2]/li[1]/span[2]/text()')
  78.             # distance = distance[0] if distance else ' '
  79.             # 滚动到想要元素
  80.             ditu = driver.find_element(By.XPATH, '//*[@id="around"]/h3')
  81.             driver.execute_script("arguments[0].scrollIntoView();", ditu)
  82.             # wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="around"]/div/div[2]/ul[2]/li[1]/p[1]/span/')))
  83.             time.sleep(1)
  84.             # driver.execute_script('window.scrollBy(0,2000)')
  85.             # js_button = 'document.documentElement.scrollTop=2400' //*[@id="around"]/h3
  86.             # driver.execute_script(js_button)
  87.             # time.sleep(2)
  88.             # js_button_ = 'document.documentElement.scrollTop=2600'
  89.             # driver.execute_script(js_button_)
  90.             # # wait.until(EC.invisibility_of_element_located((By.XPATH, '//*[@id="around"]/ul[1]/li[2]')))
  91.             # time.sleep(2)
  92.             # js_button_ = 'document.documentElement.scrollTop=300'
  93.             # driver.execute_script(js_button_)
  94.             # time.sleep(0.5)
  95.             # 地铁
  96.             tree = etree.HTML(driver.page_source)
  97.             metro = tree.xpath('//*[@id="around"]/div/div[2]/ul[2]/li/p[1]/span/text()')
  98.             metro = metro[0] if metro else ''
  99.             # 公交
  100.             button = driver.find_element(By.XPATH, '//*[@id="around"]/div/div[2]/ul[1]/li[2]')
  101.             driver.execute_script("$(arguments[0]).click()", button)
  102.             time.sleep(1)
  103.             tree = etree.HTML(driver.page_source)
  104.             bus_route = tree.xpath('//*[@id="around"]/div/div[2]/ul[2]/li[1]/p[2]/text()')
  105.             bus_route = bus_route[0] if bus_route else ''
  106.             # 学校
  107.             button = driver.find_element(By.XPATH, '//*[@id="around"]/div/div[2]/ul[1]/li[3]')
  108.             driver.execute_script("$(arguments[0]).click()", button)
  109.             time.sleep(2)
  110.             tree = etree.HTML(driver.page_source)
  111.             school = tree.xpath('//*[@id="around"]/div/div[2]/ul[2]/li[1]/p[1]/span/text()')
  112.             school = str(school).strip('[').strip(']')
  113.             # 医院
  114.             button = driver.find_element(By.XPATH, '//*[@id="around"]/div/div[2]/ul[1]/li[4]')
  115.             driver.execute_script("$(arguments[0]).click()", button)
  116.             time.sleep(2)
  117.             tree = etree.HTML(driver.page_source)
  118.             hospital = tree.xpath('//*[@id="around"]/div/div[2]/ul[2]/li[1]/p[1]/span/text()')
  119.             hospital = str(hospital).strip('[').strip(']')
  120.             # # 购物
  121.             # button = driver.find_element(By.XPATH, '//*[@id="around"]/div/div[2]/ul[1]/li[7]')
  122.             # driver.execute_script("$(arguments[0]).click()", button)
  123.             # time.sleep(2)
  124.             # tree = etree.HTML(driver.page_source)
  125.             # supermarket = tree.xpath('//*[@id="around"]/div/div[2]/ul[2]/li/p[2]/text()')
  126.             # supermarket = str(supermarket).strip('[').strip(']')
  127.  
  128.             data_list = [title, price, house_class, area, maintain, floor, parking_lot, use_cable, heating, point,
  129.                          move_into_time, lift, use_water, gas, tenancy,
  130.                          status1, status2, status3, status4, status5, status6, status7, status8, status9, status10,
  131.                          metro, bus_route, school,hospital,url
  132.                          ]
  133.             print(data_list)
  134.             with open('used_url.txt','a+',encoding='utf-8') as fp:
  135.                 fp.write(url+'\n')
  136.         except:
  137.            print(f'出错url:{url}')
  138.     return data_list

总结

点开主页可以下载全部代码跟数据进行参考！

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/不正经/article/detail/212254
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号