py爬虫资料
内容导读
互联网集市收集整理的这篇技术教程文章主要介绍了py爬虫资料，小编现在分享给大家，供广大互联网技能从业者学习和参考。文章包含9928字，纯文字阅读大概需要15分钟。
内容图文
Requests
Python标准库中提供了：urllib、urllib2、httplib等模块以供Http请求，但是，它的 API 太渣了。它是为另一个时代、另一个互联网所创建的。它需要巨量的工作，甚至包括各种方法覆盖，来完成最简单的任务。
Requests 是使用 Apache2 Licensed 许可证的 基于Python开发的HTTP 库，其在Python内置模块的基础上进行了高度的封装，从而使得Pythoner进行网络请求时，变得美好了许多，使用Requests可以轻而易举的完成浏览器可有的任何操作。
1、get请求
#
1、无参数实例
import
requests
ret = requests.get(‘https://github.com/timeline.json‘)
# 2、有参数实例import requests
payload = {‘key1‘: ‘value1‘, ‘key2‘: ‘value2‘}
ret = requests.get("http://httpbin.org/get", params=payload)
2、post请求
#
1、基本POST实例
import
requests
payload = {‘key1‘: ‘value1‘, ‘key2‘: ‘value2‘}
ret = requests.post("http://httpbin.org/post", data=payload)
# 2、发送请求头和数据实例import requests
import json
url = ‘https://api.github.com/some/endpoint‘
payload = {‘some‘: ‘data‘}
headers = {‘content-type‘: ‘application/json‘}
ret = requests.post(url, data=json.dumps(payload), headers=headers)
def
param_param():
#
- 可以是字典
#
- 可以是字符串
#
- 可以是字节（ascii编码以内）
#
requests.request(method=‘get‘,
#
url=‘http://127.0.0.1:8000/test/‘,
#
params={‘k1‘: ‘v1‘, ‘k2‘: ‘水电费‘})
#
requests.request(method=‘get‘,
#
url=‘http://127.0.0.1:8000/test/‘,
#
params="k1=v1&k2=水电费&k3=v3&k3=vv3")
#
requests.request(method=‘get‘,
#
url=‘http://127.0.0.1:8000/test/‘,
#
params=bytes("k1=v1&k2=k2&k3=v3&k3=vv3", encoding=‘utf8‘))
pass
def
param_data():
#
可以是字典
#
可以是字符串
#
可以是字节
#
可以是文件对象
#
requests.request(method=‘POST‘,
#
url=‘http://127.0.0.1:8000/test/‘,
#
data={‘k1‘: ‘v1‘, ‘k2‘: ‘水电费‘})
#
requests.request(method=‘POST‘,
#
url=‘http://127.0.0.1:8000/test/‘,
#
data="k1=v1; k2=v2; k3=v3; k3=v4"
#
)
#
requests.request(method=‘POST‘,
#
url=‘http://127.0.0.1:8000/test/‘,
#
data="k1=v1;k2=v2;k3=v3;k3=v4",
#
headers={‘Content-Type‘: ‘application/x-www-form-urlencoded‘}
#
)
#
requests.request(method=‘POST‘,
#
url=‘http://127.0.0.1:8000/test/‘,
#
data=open(‘data_file.py‘, mode=‘r‘, encoding=‘utf-8‘), # 文件内容是：k1=v1;k2=v2;k3=v3;k3=v4
#
headers={‘Content-Type‘: ‘application/x-www-form-urlencoded‘}
#
)
pass
def
param_json():
#
将json中对应的数据进行序列化成一个字符串，json.dumps(...)
#
然后发送到服务器端的body中，并且Content-Type是 {‘Content-Type‘: ‘application/json‘}
#
设置编码
#
requests.post(init_url,data=json.dumps(post_data),headers={‘Content-Type‘:‘application/json;charset=utf-8‘})
requests.request(method=‘POST‘,
url=‘http://127.0.0.1:8000/test/‘,
json={‘k1‘: ‘v1‘, ‘k2‘: ‘水电费‘})
def param_headers():
# 发送请求头到服务器端
requests.request(method=‘POST‘,
url=‘http://127.0.0.1:8000/test/‘,
json={‘k1‘: ‘v1‘, ‘k2‘: ‘水电费‘},
headers={‘Content-Type‘: ‘application/x-www-form-urlencoded‘}
)
def param_cookies():
# 发送Cookie到服务器端
requests.request(method=‘POST‘,
url=‘http://127.0.0.1:8000/test/‘,
data={‘k1‘: ‘v1‘, ‘k2‘: ‘v2‘},
cookies={‘cook1‘: ‘value1‘},
)
# 也可以使用CookieJar（字典形式就是在此基础上封装）from http.cookiejar import CookieJar
from http.cookiejar import Cookie
obj = CookieJar()
obj.set_cookie(Cookie(version=0, name=‘c1‘, value=‘v1‘, port=None, domain=‘‘, path=‘/‘, secure=False, expires=None,
discard=True, comment=None, comment_url=None, rest={‘HttpOnly‘: None}, rfc2109=False,
port_specified=False, domain_specified=False, domain_initial_dot=False, path_specified=False)
)
requests.request(method=‘POST‘,
url=‘http://127.0.0.1:8000/test/‘,
data={‘k1‘: ‘v1‘, ‘k2‘: ‘v2‘},
cookies=obj)
def param_files():
# 发送文件# file_dict = {# ‘f1‘: open(‘readme‘, ‘rb‘)# }# requests.request(method=‘POST‘,# url=‘http://127.0.0.1:8000/test/‘,# files=file_dict)# 发送文件，定制文件名# file_dict = {# ‘f1‘: (‘test.txt‘, open(‘readme‘, ‘rb‘))# }# requests.request(method=‘POST‘,# url=‘http://127.0.0.1:8000/test/‘,# files=file_dict)# 发送文件，定制文件名# file_dict = {# ‘f1‘: (‘test.txt‘, "hahsfaksfa9kasdjflaksdjf")# }# requests.request(method=‘POST‘,# url=‘http://127.0.0.1:8000/test/‘,# files=file_dict)# 发送文件，定制文件名# file_dict = {# ‘f1‘: (‘test.txt‘, "hahsfaksfa9kasdjflaksdjf", ‘application/text‘, {‘k1‘: ‘0‘})# }# requests.request(method=‘POST‘,# url=‘http://127.0.0.1:8000/test/‘,# files=file_dict)passdef param_auth():
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
ret = requests.get(‘https://api.github.com/user‘, auth=HTTPBasicAuth(‘wupeiqi‘, ‘sdfasdfasdf‘))
print(ret.text)
# ret = requests.get(‘http://192.168.1.1‘,# auth=HTTPBasicAuth(‘admin‘, ‘admin‘))# ret.encoding = ‘gbk‘# print(ret.text)# ret = requests.get(‘http://httpbin.org/digest-auth/auth/user/pass‘, auth=HTTPDigestAuth(‘user‘, ‘pass‘))# print(ret)#
def param_timeout():
# ret = requests.get(‘http://google.com/‘, timeout=1)# print(ret)# ret = requests.get(‘http://google.com/‘, timeout=(5, 1))# print(ret)passdef param_allow_redirects():
ret = requests.get(‘http://127.0.0.1:8000/test/‘, allow_redirects=False)
print(ret.text)
def param_proxies():
# proxies = {# "http": "61.172.249.96:80",# "https": "http://61.185.219.126:3128",# }# proxies = {‘http://10.20.1.128‘: ‘http://10.10.1.10:5323‘}# ret = requests.get("http://www.proxy360.cn/Proxy", proxies=proxies)# print(ret.headers)# from requests.auth import HTTPProxyAuth#
# proxyDict = {# ‘http‘: ‘77.75.105.165‘,# ‘https‘: ‘77.75.105.165‘# }# auth = HTTPProxyAuth(‘username‘, ‘mypassword‘)#
# r = requests.get("http://www.google.com", proxies=proxyDict, auth=auth)# print(r.text)passdef param_stream():
ret = requests.get(‘http://127.0.0.1:8000/test/‘, stream=True)
print(ret.content)
ret.close()
# from contextlib import closing# with closing(requests.get(‘http://httpbin.org/get‘, stream=True)) as r:# # 在此处理响应。# for i in r.iter_content():# print(i)def requests_session():
import requests
session = requests.Session()
### 1、首先登陆任何页面，获取cookie
i1 = session.get(url="http://dig.chouti.com/help/service")
### 2、用户登陆，携带上一次的cookie，后台对cookie中的 gpsd 进行授权
i2 = session.post(
url="http://dig.chouti.com/login",
data={
‘phone‘: "8615131255089",
‘password‘: "xxxxxx",
‘oneMonth‘: ""
}
)
i3 = session.post(
url="http://dig.chouti.com/link/vote?linksId=8589623",
)
print(i3.text)
BeautifulSoup
BeautifulSoup是一个模块，该模块用于接收一个HTML或XML字符串，然后将其进行格式化，之后遍可以使用他提供的方法进行快速查找指定元素，从而使得在HTML或XML中查找指定元素变得简单。
from bs4 import BeautifulSoup
html_doc = """
<html><head><title>The Dormouse‘s story</title></head>
<body>
asdf
<div class="title">
<b>The Dormouse‘s story总共</b>
<h1>f</h1>
</div>
<div class="story">Once upon a time there were three little sisters; and their names were
<a class="sister0" id="link1">Els<span>f</span>ie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</div>
ad<br/>sf
<p class="story">...</p>
</body>
</html>
"""
soup = BeautifulSoup(html_doc, features="lxml")
# 找到第一个a标签
tag1 = soup.find(name=‘a‘)
# 找到所有的a标签
tag2 = soup.find_all(name=‘a‘)
# 找到id＝link2的标签
tag3 = soup.select(‘#link2‘)
安装：
pip3 install beautifulsoup4
1. name，标签名称
#
tag = soup.find(‘a‘)
#
name = tag.name # 获取
#
print(name)
#
tag.name = ‘span‘ # 设置
#
print(soup)
2. attr，标签属性
#
tag = soup.find(‘a‘)
#
attrs = tag.attrs # 获取
#
print(attrs)
#
tag.attrs = {‘ik‘:123} # 设置
#
tag.attrs[‘id‘] = ‘iiiii‘ # 设置
#
print(soup)
3. children,所有子标签
#
body = soup.find(‘body‘)
#
v = body.children
4. children,所有子子孙孙标签
#
body = soup.find(‘body‘)
#
v = body.descendants
5. clear,将标签的所有子标签全部清空（保留标签名）
#
tag = soup.find(‘body‘)
#
tag.clear()
#
print(soup)
6. decompose,递归的删除所有的标签
#
body = soup.find(‘body‘)
#
body.decompose()
#
print(soup)
7. extract,递归的删除所有的标签，并获取删除的标签
#
body = soup.find(‘body‘)
#
v = body.extract()
#
print(soup)
8. decode,转换为字符串（含当前标签）；decode_contents（不含当前标签）
#
body = soup.find(‘body‘)
#
v = body.decode()
#
v = body.decode_contents()
#
print(v)
9. encode,转换为字节（含当前标签）；encode_contents（不含当前标签）
#
body = soup.find(‘body‘)
#
v = body.encode()
#
v = body.encode_contents()
#
print(v)
10. find,获取匹配的第一个标签
#
tag = soup.find(‘a‘)
#
print(tag)
#
tag = soup.find(name=‘a‘, attrs={‘class‘: ‘sister‘}, recursive=True, text=‘Lacie‘)
#
tag = soup.find(name=‘a‘, class_=‘sister‘, recursive=True, text=‘Lacie‘)
#
print(tag)
11. find_all,获取匹配的所有标签
#
tags = soup.find_all(‘a‘)
#
print(tags)
#
tags = soup.find_all(‘a‘,limit=1)
#
print(tags)
#
tags = soup.find_all(name=‘a‘, attrs={‘class‘: ‘sister‘}, recursive=True, text=‘Lacie‘)
#
# tags = soup.find(name=‘a‘, class_=‘sister‘, recursive=True, text=‘Lacie‘)
#
print(tags)
#
####### 列表 #######
#
v = soup.find_all(name=[‘a‘,‘div‘])
#
print(v)
#
v = soup.find_all(class_=[‘sister0‘, ‘sister‘])
#
print(v)
#
v = soup.find_all(text=[‘Tillie‘])
#
print(v, type(v[0]))
#
v = soup.find_all(id=[‘link1‘,‘link2‘])
#
print(v)
#
v = soup.find_all(href=[‘link1‘,‘link2‘])
#
print(v)
#
####### 正则 #######
import
re
#
rep = re.compile(‘p‘)
#
rep = re.compile(‘^p‘)
#
v = soup.find_all(name=rep)
#
print(v)
#
rep = re.compile(‘sister.*‘)
#
v = soup.find_all(class_=rep)
#
print(v)
#
rep = re.compile(‘http://www.oldboy.com/static/.*‘)
#
v = soup.find_all(href=rep)
#
print(v)
#
####### 方法筛选 #######
#
def func(tag):
#
return tag.has_attr(‘class‘) and tag.has_attr(‘id‘)
#
v = soup.find_all(name=func)
#
print(v)
#
## get,获取标签属性
#
tag = soup.find(‘a‘)
#
v = tag.get(‘id‘)
#
print(v)
12. has_attr,检查标签是否具有该属性
#
tag = soup.find(‘a‘)
#
v = tag.has_attr(‘id‘)
#
print(v)
13. get_text,获取标签内部文本内容
#
tag = soup.find(‘a‘)
#
v = tag.get_text(‘id‘)
#
print(v)
14. index,检查标签在某标签中的索引位置
#
tag = soup.find(‘body‘)
#
v = tag.index(tag.find(‘div‘))
#
print(v)
#
tag = soup.find(‘body‘)
#
for i,v in enumerate(tag):
#
print(i,v)
15. is_empty_element,是否是空标签(是否可以是空)或者自闭合标签，
判断是否是如下标签：‘br‘ , ‘hr‘, ‘input‘, ‘img‘, ‘meta‘,‘spacer‘, ‘link‘, ‘frame‘, ‘base‘
#
tag = soup.find(‘br‘)
#
v = tag.is_empty_element
#
print(v)
16. 当前的关联标签
#
soup.next
#
soup.next_element
#
soup.next_elements
#
soup.next_sibling
#
soup.next_siblings
#
#
tag.previous
#
tag.previous_element
#
tag.previous_elements
#
tag.previous_sibling
#
tag.previous_siblings
#
#
tag.parent
#
tag.parents
原文：http://www.cnblogs.com/wangyufu/p/7749360.html
内容总结
以上是互联网集市为您收集整理的py爬虫资料全部内容，希望文章能够帮你解决py爬虫资料所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错，欢迎将互联网集市网站推荐给程序员好友。
内容备注
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 gblab@vip.qq.com 举报，一经查实，本站将立刻删除。
内容手机端
扫描二维码推送至手机访问。
来源：【匿名】