python爬取今日头条关键字图集
内容导读
互联网集市收集整理的这篇技术教程文章主要介绍了python爬取今日头条关键字图集，小编现在分享给大家，供广大互联网技能从业者学习和参考。文章包含5698字，纯文字阅读大概需要9分钟。
内容图文
１．访问搜索图集结果，获得json如下(右图为data的一条的详细内容)．页面以Ajax呈现，每次请求20个图集，其中
title --- 图集名字
artical_url --- 图集的地址
count --- 图集图片数量
２． 访问其中的图集
访问artical_url，获得图集图片详细信息，其中图片url为下载地址
展现出爬虫关键部分，整体项目地址在https://github.com/GeoffreyHub/toutiao_spider
1 #!/usr/bin/env python
2 # encoding: utf-8
3
4 """
5 @version: python37
6 @author: Geoffrey
7 @file: spider.py
8 @time: 18-10-24 上午11:15
9 """
10 import json
11 import re
12 from multiprocessing import Pool
13 import urllib3
14 urllib3.disable_warnings()
15 from requests import RequestException
16
17 from common.request_help import make_session
18 from db.mysql_handle import MysqlHandler
19 from img_spider.settings import *
20
21
22
23 class SpiderTouTiao:
24
25
26 def __init__(self, keyword):
27 self.session = make_session(debug=True)
28 self.url_index = 'https://www.toutiao.com/search_content/'
29 self.keyword = keyword
30 self.mysql_handler = MysqlHandler(MYSQL_CONFIG)
31
32 def search_index(self, offset):
33 url = self.url_index
34 data = {
35 'offset': f'{offset}',
36 'format': 'json',
37 'keyword': self.keyword,
38 'autoload': 'true',
39 'count': '20',
40 'cur_tab': '3',
41 'from': 'gallery'
42 }
43
44 try:
45 response = self.session.get(url, params=data)
46 if response.status_code is 200:
47 json_data = response.json()
48 with open(f'../json_data/搜索结果-{offset}.json', 'w', encoding='utf-8') as f:
49 json.dump(json_data, f, indent=4, ensure_ascii=False)
50 return self.get_gallery_url(json_data)
51 except :
52 pass
53 print('请求失败')
54
55 @staticmethod
56 def get_gallery_url(json_data):
57 dict_data = json.dumps(json_data)
58 for info in json_data["data"]:
59 title = info["title"]
60 gallery_pic_count = info["gallery_pic_count"]
61 article_url = info["article_url"]
62 yield title, gallery_pic_count, article_url
63
64 def gallery_list(self, search_data):
65 gallery_urls = {}
66 for title, gallery_pic_count, article_url in search_data:
67 print(title, gallery_pic_count, article_url)
68 response = self.session.get(article_url)
69 html = response.text
70 images_pattern = re.compile('gallery: JSON.parse\("(.*?)"\),', re.S)
71 result = re.search(images_pattern, html)
72
73 if result:
74 # result = result.replace('\\', '')
75 # result = re.sub(r"\\", '', result)
76 result = eval("'{}'".format(result.group(1)))
77 result = json.loads(result)
78 # picu_urls = zip(result["sub_abstracts"], result["sub_titles"], [url["url"] for url in result["sub_images"]])
79 picu_urls = zip(result["sub_abstracts"], [url["url"] for url in result["sub_images"]])
80 # print(list(picu_urls))
81 gallery_urls[title] = picu_urls
82 else:
83 print('解析不到图片ｕrl')
84
85 with open(f'../json_data/{title}-搜索结果.json', 'w', encoding='utf-8') as f:
86 json.dump(result, f, indent=4, ensure_ascii=False)
87
88 break
89
90 # print(gallery_urls)
91 return gallery_urls
92
93 def get_imgs(self, gallery_urls):
94 params = []
95 for title, infos in (gallery_urls.items()):
96 for index, info in enumerate(infos):
97 abstract, img_url = info
98 print(index, abstract)
99 response = self.session.get(img_url)
100 img_content = response.content
101 params.append([title, abstract, img_content])
102
103 with open(f'/home/geoffrey/图片/今日头条/{title}-{index}.jpg', 'wb') as f:
104 f.write(img_content)
105
106 SQL = 'insert into img_gallery(title, abstract, imgs) values(%s, %s, %s)'
107 self.mysql_handler.insertOne(SQL, [title, abstract, img_content])
108 self.mysql_handler.end()
109
110 print(f'保存图集完成' + '-'*50 )
111 # SQL = 'insert into img_gallery(title, abstract, imgs) values(%s, %s, %s)'
112 # self.mysql_handler.insertMany(SQL, params)
113 # self.mysql_handler.end()
114
115
116 def main(offset):
117 spider = SpiderTouTiao(KEY_WORD)
118 search_data = spider.search_index(offset)
119 gallery_urls = spider.gallery_list(search_data)
120 spider.get_imgs(gallery_urls)
121 spider.mysql_handler.dispose()
122
123
124 if __name__ == '__main__':
125 groups = [x*20 for x in range(GROUP_START, GROPE_END)]
126
127 pool = Pool(10)
128 pool.map(main, groups)
129
130 # for i in groups:
131 # main(i)
项目结构如下：
.
├── common
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-37.pyc
│ │ └── request_help.cpython-37.pyc
│ ├── request_help.py
├── db
│ ├── __init__.py
│ ├── mysql_handle.py
│ └── __pycache__
│ ├── __init__.cpython-37.pyc
│ └── mysql_handle.cpython-37.pyc
├── img_spider
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-37.pyc
│ │ └── settings.cpython-37.pyc
│ ├── settings.py
│ └── spider.py
└── json_data
├── 沐浴三里屯的秋-搜索结果.json
├── 盘点三里屯那些高逼格的苍蝇馆子-搜索结果.json
├── 搜索结果-0.json
├── 搜索结果-20.json
├── 搜索结果-40.json
内容总结
以上是互联网集市为您收集整理的python爬取今日头条关键字图集全部内容，希望文章能够帮你解决python爬取今日头条关键字图集所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错，欢迎将互联网集市网站推荐给程序员好友。
内容备注
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 gblab@vip.qq.com 举报，一经查实，本站将立刻删除。
内容手机端
扫描二维码推送至手机访问。