分享一个简单的python+mysql网络数据抓取

             1
            #
             -*- coding: UTF-8 -*-
             2
             3
            from bs4 import BeautifulSoup
 4import threading
 5import requests
 6import pymysql
 7import string
 8import time
 9import sys
10import re
111213#爬取频率，单位：秒14 SECONDS_PER_CRAWL = 10
1516#更新到数据库频率，单位：爬取次数17 CRAWL_PER_UPDATE_TO_DB = 1
1819#爬取目标网站url20 CRAWL_TARGET_URL = ‘http://top.baidu.com/‘2122class DataProducer(threading.Thread):
2324#临时存放爬取结果25     NewsDict = {}
2627def__init__(self):
28         threading.Thread.__init__(self)
29         db = pymysql.connect(host="localhost", user="root", port=3306, passwd=None, db="crawler", charset="utf8")
30         cursor = db.cursor()
31         sql = """CREATE TABLE IF NOT EXISTS HOTNEWS (
32              INFORMATION VARCHAR(64) NOT NULL,
33              HYPERLINK      VARCHAR(256),
34              PRIMARY KEY (INFORMATION) );"""35        cursor.execute(sql)
36        db.close()
3738def run(self):
39print("DataProducer Thread start!")
40        crawl_data(self.NewsDict)
41print("DataProducer Thread exit!")
424344def crawl_data(nd):
45     count = 0;
46while 1:
47         req = requests.get(url=CRAWL_TARGET_URL)
48         req.encoding = req.apparent_encoding
49         bf = BeautifulSoup(req.text, "html.parser")
50         texts = bf.find_all(‘ul‘, id="hot-list", class_="list")
51         bfs = BeautifulSoup(str(texts), "html.parser")
52         spans = bfs.find_all(‘span‘, class_ = re.compile("icon-fall|icon-rise|icon-fair"))
53         lis = bfs.find_all(‘a‘, class_ = "list-title")
54for i in range(10):
55             vtup = (spans[i].get_text(), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
56             ktup = (lis[i].get(‘title‘), lis[i].get(‘href‘))
57if ktup in nd.keys():
58                nd[ktup].append(vtup)
59else:
60                 nd[ktup] = [vtup]
61         count = count+1
62if count%CRAWL_PER_UPDATE_TO_DB == 0:
63            update_to_db(nd)
64            nd.clear()
65        time.sleep(SECONDS_PER_CRAWL)
6667def update_to_db(nd):
68     db = pymysql.connect(host="localhost", user="root", port=3306, passwd=None, db="crawler", charset="utf8")
69     cursor = db.cursor()
70for k in nd.keys():
71#将热点插入至主表72         sql1 = "REPLACE INTO HOTNEWS (INFORMATION, HYPERLINK) VALUES ( ‘%s‘, ‘%s‘);"73#热点创建其分表74         sql2 = "CREATE TABLE IF NOT EXISTS `%s`( NUMBEROFPEOPLE INT NOT NULL, OCCURTIME DATETIME, PRIMARY KEY (OCCURTIME) );"75try:
76             cursor.execute(sql1%(k[0], k[1]))
77             cursor.execute(sql2%(k[0]))
78            db.commit()
79except:
80            db.rollback()
81#将每个热点数据插入到对应热点分表82for e in nd[k]:
83             insert_sql = "INSERT INTO %s (NUMBEROFPEOPLE, OCCURTIME) VALUES (%d, ‘%s‘);"84try:
85                 cursor.execute(insert_sql%(k[0], int(e[0]), e[1]))
86except:
87                db.rollback()
88    db.close()
8990if__name__ == ‘__main__‘:
91     t = DataProducer()
92    t.start()
93     t.join()