Python 爬取大众点评店铺评论
内容导读
互联网集市收集整理的这篇技术教程文章主要介绍了Python 爬取大众点评店铺评论，小编现在分享给大家，供广大互联网技能从业者学习和参考。文章包含4620字，纯文字阅读大概需要7分钟。
内容图文
这是我自己写的代码，有可以优化的地方请不要藏着，要告诉我哦！
import parsel
import pymysql
from lxml import etree
import re
import requests
def download_data(url,cookie):
'''
获取加密网页源码
获取加密文件
:return:
'''
headers = {
"Cookie": cookie,
"Referer": "http://www.dianping.com/",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36"
}
'''
获取原始网页
'''
ret = requests.get(url=url, headers=headers).text
with open('01 原始网页_加密.html', 'w', encoding='utf-8') as f:
f.write(ret)
'''
获取css文件
'''
css_url = re.findall('<link rel="stylesheet" type="text/css" href="(//s3plus.meituan.*?)">', ret)
css_url = 'https:' + css_url[0]
css_response = requests.get(css_url).text
with open('02 css样式.css', 'w', encoding='utf-8') as f:
f.write(css_response)
'''
获取svg对照表
'''
svg_urls = re.findall(r'.*?\[class\^="(.*?)"\]\{.*?background-image: url\((.*?)\);', css_response)
for svg_url in svg_urls:
name, url = svg_url
svg_url = 'https:' + url
svg_response = requests.get(svg_url).text
with open(F'03 svg对照表{name}.svg', 'w', encoding='utf-8') as f:
f.write(svg_response)
def crack_data():
'''
解密数据，破解svg对应关系
:return:
'''
with open('03 svg对照表zpd.svg', 'r', encoding='utf-8') as f:#文件名称根据获取到的svg文件更换
svg_html = f.read()
sel = parsel.Selector(svg_html)
texts = sel.css('textPath')
paths = sel.css('path')
path_dict = {}
for path in paths:
path_dict[path.css('path::attr(id)').get()] = path.css('path::attr(d)').get().split(' ')[1]
# print(path.css('path::attr(id)').get())
# print(path.css('path::attr(d)').get().split(' ')[1])
count = 1
zpd_svg_dict = {} # y坐标和字符串的联系
for text in texts:
zpd_svg_dict[path_dict[str(count)]] = text.css('textPath::text').get()
count += 1
print(zpd_svg_dict)
with open('02 css样式.css', 'r', encoding='utf-8') as f:
css_html = f.read()
css_paths = re.findall(r'''
\.(zpd.*?) {
background: -(\d+)\.0px -(\d+)\.0px;
\}
''', css_html) # 正则表达式条件根据css文件类标签更换
print(css_paths)
last_map = {}
for css_path in css_paths:
css_name, x, y = css_path
index = int(int(x) / 14)
for i in zpd_svg_dict:
if int(y) > int(i):
pass
else:
last_map[css_name] = zpd_svg_dict[i][index]
break
return last_map
def decryption(last_map):
'''
返回破解后的html
:param last_map:
:return:
'''
with open('01 原始网页_加密.html', 'r', encoding='utf-8') as f:
ret = f.read()
svg_list = re.findall('<svgmtsi class="(.*?)"></svgmtsi>', ret)
for svg in svg_list:
print(svg, last_map[svg])
ret = ret.replace(f'<svgmtsi class="{svg}"></svgmtsi>', last_map[svg])
return ret
def write_data(ret):
'''
获取评论数据并写入数据库
:param ret:
:return:
'''
# 用不到的div标签去掉 并不是全部都有这个标签 影响代码编写
ret = ret.replace(' <div class="richtitle">消费后评价</div>', '')
# ret = ret.replace(div,'')
# print(ret)
etre = etree.HTML(ret)
li_list = etre.xpath('//*[@id="review-list"]/div[2]/div[3]/div[3]/div[3]/ul/li')
# 初始化数据库
db = pymysql.connect(host='localhost', user='root', password='123456', port=3306, database='review',
charset='utf8mb4')
cursor = db.cursor()
count = 0
for li in li_list:
name = li.xpath('./div[@class="main-review"]/div[1]/a/text()')[0].strip()
score = re.findall('sml-rank-stars sml-str(.*?) star', li.xpath('./div[1]/div[2]/span[1]/@class')[0])[0].strip()
time = li.xpath('//div[@class="misc-info clearfix"]/span[1]/text()')[count].strip()
shop_name = li.xpath('//div[@class="misc-info clearfix"]/span[2]/text()')[count].strip()
comment = ','.join([i.replace('\n', '').strip() for i in li.xpath('./div/div[4]/text()')])
count += 1
print(name, score, time, shop_name, comment)
# 写入数据库
sql = 'insert into dianping(name,score,time,shop_name,comment)values(%s,%s,%s,%s,%s)'
cursor.execute(sql, (name, score, time, shop_name, comment))
db.commit()
# 关闭连接
db.close()
if __name__ == '__main__':
#cookie 不定时更换
cookie = "s_ViewType=10; _lxsdk_cuid=175e331ad79c8-0996df2d570671-46460e2a-1fa400-175e331ad79c8; _lxsdk=175e331ad79c8-0996df2d570671-46460e2a-1fa400-175e331ad79c8; _hc.v=c4dfac1c-01af-6a87-d803-2cd6b8db107a.1605834485; fspop=test; ctu=ef0b64e4cabf67f148563284ea8c8d0555a008f7ca0dee097831c90b52822812; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1605834487,1605835298,1606093773; cy=2; cye=beijing; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1606098153; expand=yes; _lxsdk_s=175f2cc7d23-6-9d5-75e%7C%7C532"
url = 'http://www.dianping.com/shop/130096343/review_all' #这是一个商家的评论 可以更换
try:
download_data(url,cookie)
except Exception:
print('出现验证码验证')#访问过多会出现验证码 目前没有破解
map_dict = {}
try:
map_dict = crack_data()
except Exception:
print('css类属性发生变化')
ret = decryption(map_dict)
write_data(ret)
内容总结
以上是互联网集市为您收集整理的Python 爬取大众点评店铺评论全部内容，希望文章能够帮你解决Python 爬取大众点评店铺评论所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错，欢迎将互联网集市网站推荐给程序员好友。
内容备注
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 gblab@vip.qq.com 举报，一经查实，本站将立刻删除。
内容手机端
扫描二维码推送至手机访问。
来源：【匿名】