devbox
代码架构师
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

爬取微博用户的相册图片_微博相册爬虫

作者：代码架构师 | 2024-02-03 14:46:48

赞

踩

微博相册爬虫

难点分析

主要是在抓包分析怎么实现分页获取，通过抓包分析，我们知道每次请求获取照片的时候，返回的数据中都会有一个since_id 和 bottom_tips_visible

当请求为最后一页数据的时候，since_id为0和bottom_tips_visible为true

用过上述分析我们就可以轻松获取数据了 

获取数据

  1. def get_page_info(username, since_id):
  2.     url = 'https://weibo.com/ajax/profile/getImageWall'
  3.     headers = {
  4.         'Cookie': 'SINAGLOBAL=666731024991.3516.1692108672688; SUB=_2A25J3_fyDeRhGeBK41IY9CbIzDyIHXVrI5m6rDV8PUJbkNANLRnTkW1NR5DGKlU-7QQXFUn8kCO7tHZWHa2lstgT; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhfqYIORlaYHk9cTmVipRvw5NHD95QcShn71KBRShM7Ws4Dqcj.i--ciK.RiKLsi--ci-88iKyFi--ci-2Ei-2ci--Ni-2EiKy8; XSRF-TOKEN=j1xKHXp5NhQe6PQfbcsrCsbK; _s_tentry=weibo.com; Apache=7785653743983.5625.1692144415457; ULV=1692144415526:3:3:3:7785653743983.5625.1692144415457:1692112430474; PC_TOKEN=7b320ce9a2; WBStorage=4d96c54e|undefined; WBPSESS=-hAVHrSkMC8S4jXFC4-lqnFhFtNlrZmAL-r5XJQqEZxHyzIHGIPV0NHIisq9ALS32gvYmuib-sMSj5_PnjsfSxrsrk42crVW65GaxDPbJhzjy5gXZg19j07R0Qy1OJfS5oZKop_lRJX1ZHkKrd5B7w==',
  5.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.203',
  6.         'Referer': 'https://weibo.com/',
  7.     }
  8.     params = {
  9.         'uid': username,
  10.         'sinceid': since_id,
  11.     }
  12.     try:
  13.         res = requests.get(url=url, headers=headers, params=params)
  14.         if res.status_code == 200:
  15.             return res.json()
  16.         return None
  17.     except requests.exceptions.RequestException as e:
  18.         print(e)
  19.         return None

解析数据

  1. def parse_json_data(json_data, username):
  2.     try:
  3.         headers = {
  4.             'Cookie': 'SINAGLOBAL=666731024991.3516.1692108672688; SUB=_2A25J3_fyDeRhGeBK41IY9CbIzDyIHXVrI5m6rDV8PUJbkNANLRnTkW1NR5DGKlU-7QQXFUn8kCO7tHZWHa2lstgT; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhfqYIORlaYHk9cTmVipRvw5NHD95QcShn71KBRShM7Ws4Dqcj.i--ciK.RiKLsi--ci-88iKyFi--ci-2Ei-2ci--Ni-2EiKy8; XSRF-TOKEN=j1xKHXp5NhQe6PQfbcsrCsbK; _s_tentry=weibo.com; Apache=7785653743983.5625.1692144415457; ULV=1692144415526:3:3:3:7785653743983.5625.1692144415457:1692112430474; PC_TOKEN=7b320ce9a2; WBStorage=4d96c54e|undefined; WBPSESS=-hAVHrSkMC8S4jXFC4-lqnFhFtNlrZmAL-r5XJQqEZxHyzIHGIPV0NHIisq9ALS32gvYmuib-sMSj5_PnjsfSxrsrk42crVW65GaxDPbJhzjy5gXZg19j07R0Qy1OJfS5oZKop_lRJX1ZHkKrd5B7w==',
  5.             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.203',
  6.             'Referer': 'https://weibo.com/',
  7.         }
  8.         img_prefix_url = 'https://wx2.sinaimg.cn/webp720/'
  9.         # pprint(json_data['data']['list'])
  10.         imgs = json_data['data']['list']
  11.         for img in imgs:
  12.             img_url = img_prefix_url + img['pid'] + '.jpg'
  13.             img_name = img['pid']
  14.             # print(img_url)
  15.             img_content = requests.get(url=img_url, headers=headers).content
  16.             if not os.path.exists(f'E:/Python/project/网络爬虫/12-爬取微博用户视频/imgs/{username}'):
  17.                 os.mkdir(f'E:/Python/project/网络爬虫/12-爬取微博用户视频/imgs/{username}')
  18.             path = img_name + '.' + 'jpg'
  19.             with open(f'imgs/{username}/{path}', mode='wb') as f:
  20.                 f.write(img_content)
  21.                 print(f'{img_name}下载完毕')
  22.         return json_data['bottom_tips_visible'], json_data['data']['since_id']
  23.     except Exception as e:
  24.         print(e)

主函数

  1. def main(username):
  2.     # username = '5252751519'
  3.     # username = '2150318380'
  4.     since_id = 0
  5.     while True:
  6.         json_data = get_page_info(username=username, since_id=since_id)
  7.         is_stop, since_id = parse_json_data(json_data=json_data, username=username)
  8.         print(f'当前状态:{is_stop}，since_id:{since_id}')
  9.         if str(is_stop) == 'True' or str(since_id) == '0':
  10.             print("已经到底了...")
  11.             break

效果展示

 完整代码

  1. import requests
  2. import os
  3. from pprint import pprint
  4.  
  5.  
  6. def get_page_info(username, since_id):
  7.     url = 'https://weibo.com/ajax/profile/getImageWall'
  8.     headers = {
  9.         'Cookie': 'SINAGLOBAL=666731024991.3516.1692108672688; SUB=_2A25J3_fyDeRhGeBK41IY9CbIzDyIHXVrI5m6rDV8PUJbkNANLRnTkW1NR5DGKlU-7QQXFUn8kCO7tHZWHa2lstgT; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhfqYIORlaYHk9cTmVipRvw5NHD95QcShn71KBRShM7Ws4Dqcj.i--ciK.RiKLsi--ci-88iKyFi--ci-2Ei-2ci--Ni-2EiKy8; XSRF-TOKEN=j1xKHXp5NhQe6PQfbcsrCsbK; _s_tentry=weibo.com; Apache=7785653743983.5625.1692144415457; ULV=1692144415526:3:3:3:7785653743983.5625.1692144415457:1692112430474; PC_TOKEN=7b320ce9a2; WBStorage=4d96c54e|undefined; WBPSESS=-hAVHrSkMC8S4jXFC4-lqnFhFtNlrZmAL-r5XJQqEZxHyzIHGIPV0NHIisq9ALS32gvYmuib-sMSj5_PnjsfSxrsrk42crVW65GaxDPbJhzjy5gXZg19j07R0Qy1OJfS5oZKop_lRJX1ZHkKrd5B7w==',
  10.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.203',
  11.         'Referer': 'https://weibo.com/',
  12.     }
  13.     params = {
  14.         'uid': username,
  15.         'sinceid': since_id,
  16.     }
  17.     try:
  18.         res = requests.get(url=url, headers=headers, params=params)
  19.         if res.status_code == 200:
  20.             return res.json()
  21.         return None
  22.     except requests.exceptions.RequestException as e:
  23.         print(e)
  24.         return None
  25.  
  26.  
  27. def parse_json_data(json_data, username):
  28.     try:
  29.         headers = {
  30.             'Cookie': 'SINAGLOBAL=666731024991.3516.1692108672688; SUB=_2A25J3_fyDeRhGeBK41IY9CbIzDyIHXVrI5m6rDV8PUJbkNANLRnTkW1NR5DGKlU-7QQXFUn8kCO7tHZWHa2lstgT; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhfqYIORlaYHk9cTmVipRvw5NHD95QcShn71KBRShM7Ws4Dqcj.i--ciK.RiKLsi--ci-88iKyFi--ci-2Ei-2ci--Ni-2EiKy8; XSRF-TOKEN=j1xKHXp5NhQe6PQfbcsrCsbK; _s_tentry=weibo.com; Apache=7785653743983.5625.1692144415457; ULV=1692144415526:3:3:3:7785653743983.5625.1692144415457:1692112430474; PC_TOKEN=7b320ce9a2; WBStorage=4d96c54e|undefined; WBPSESS=-hAVHrSkMC8S4jXFC4-lqnFhFtNlrZmAL-r5XJQqEZxHyzIHGIPV0NHIisq9ALS32gvYmuib-sMSj5_PnjsfSxrsrk42crVW65GaxDPbJhzjy5gXZg19j07R0Qy1OJfS5oZKop_lRJX1ZHkKrd5B7w==',
  31.             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.203',
  32.             'Referer': 'https://weibo.com/',
  33.         }
  34.         img_prefix_url = 'https://wx2.sinaimg.cn/webp720/'
  35.         # pprint(json_data['data']['list'])
  36.         imgs = json_data['data']['list']
  37.         for img in imgs:
  38.             img_url = img_prefix_url + img['pid'] + '.jpg'
  39.             img_name = img['pid']
  40.             # print(img_url)
  41.             img_content = requests.get(url=img_url, headers=headers).content
  42.             if not os.path.exists(f'E:/Python/project/网络爬虫/12-爬取微博用户视频/imgs/{username}'):
  43.                 os.mkdir(f'E:/Python/project/网络爬虫/12-爬取微博用户视频/imgs/{username}')
  44.             path = img_name + '.' + 'jpg'
  45.             with open(f'imgs/{username}/{path}', mode='wb') as f:
  46.                 f.write(img_content)
  47.                 print(f'{img_name}下载完毕')
  48.         return json_data['bottom_tips_visible'], json_data['data']['since_id']
  49.     except Exception as e:
  50.         print(e)
  51.  
  52.  
  53. def main(username):
  54.     # username = '5252751519'
  55.     # username = '2150318380'
  56.     since_id = 0
  57.     while True:
  58.         json_data = get_page_info(username=username, since_id=since_id)
  59.         is_stop, since_id = parse_json_data(json_data=json_data, username=username)
  60.         print(f'当前状态:{is_stop}，since_id:{since_id}')
  61.         if str(is_stop) == 'True' or str(since_id) == '0':
  62.             print("已经到底了...")
  63.             break
  64.  
  65.  
  66. if __name__ == '__main__':
  67.     username = input("请输入您要爬取的用户id:")
  68.     main(username=username)

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/article/detail/56971
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号