Python3.5爬取豆瓣电视剧数据并且同步到mysql中

内容导读

互联网集市收集整理的这篇技术教程文章主要介绍了Python3.5爬取豆瓣电视剧数据并且同步到mysql中，小编现在分享给大家，供广大互联网技能从业者学习和参考。文章包含6181字，纯文字阅读大概需要9分钟。

内容图文

!/usr/local/bin/python # -*- coding: utf-8 -*- # Python: 3.5 # Author: zhenghai.zhang@xxx.com # Program: 爬取豆瓣网站上所有电视剧的名称并写入数据库。 # Version: 0.1 # History: 2017.11.01 import requests,time, pymysql, re, datetime, json from exchangelib import DELEGATE, Account, Credentials, Message, Mailbox, HTMLBody host = ‘xxx‘ user = ‘xxx‘ passwd = ‘xxx‘ dbme = ‘crawl‘ dbtarget = ‘back_brace‘ table = ‘tv_hotwords‘ tabledelta = ‘tv_hotwords_delta‘ tablesync = ‘slot_value‘ port = 3306 tolist = [‘zhenghai.zhang@xxx.com‘] def get_tvs(urlbase, page): try: url = urlbase + str(page) print(url) try: result = requests.get(url).text jresult = json.loads(result) tvs = jresult.get(‘subjects‘) except: print(‘爬取‘ + urlbase + str(page) + ‘失败！‘) time.sleep(2) return tvs except: print(‘获取第%s页电影列表失败‘ % page) def tv_insert(host, user, passwd, dbme, port, table, tvs_list): conn = pymysql.connect(host=host, user=user, passwd=passwd, db=dbme, port=port, charset="utf8") cur = conn.cursor() new_tvs = [] punc = "！？?＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～?????、〃》「」『』【】〔〕〖〗?????〝〞????–—‘’?“”??…?﹏.():：。· " punctuation = punc for tv in tvs_list: try: tv[‘title‘] = re.sub(r"[%s]+" % punctuation, "", tv.get(‘title‘)) cmd = ‘insert into %s(tv_id, tv_name) values("%s", "%s")‘ % ( table, tv.get(‘id‘), tv.get(‘title‘)) cur.execute(cmd) new_tvs.append(tv) except: print(" " * 20, tv.get(‘title‘), "already exists, skip……") cur.close() conn.commit() conn.close() return new_tvs def tv_new_and_sync(host, user, passwd, dbme, dbtarget, port, tabledelta, tvs_list, tablesync): conn = pymysql.connect(host=host, user=user, passwd=passwd, db=dbme, port=port, charset="utf8") cur = conn.cursor() cur.execute("delete from %s " % dbme+"."+tabledelta) for tv in tvs_list: try: cmd = ‘insert into %s(tv_id, tv_name) values("%s", "%s")‘ % (tabledelta, tv[‘id‘], tv[‘title‘]) cmdsync = ‘insert into %s(slot_type_id, slot_value, create_by, modify_by, gmt_create, gmt_modify, out_value) values("%s", "%s", "%s", "%s", "%s", "%s", "%s")‘ % (dbtarget+"."+tablesync, "xxxxxx", tv[‘title‘], "system", "system", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "") cur.execute(cmd) cur.execute(cmdsync) except pymysql.Error: print(" " * 20, tv[‘title‘], "already exists, skip……") cur.close() conn.commit() conn.close() def tv_new_to_release(host, user, passwd, dbtarget, port): conn = pymysql.connect(host=host, user=user, passwd=passwd, db=dbtarget, port=port, charset="utf8") cur = conn.cursor() try: cmdbacktoskill = ‘insert into back_brace.release_task(app_type,app_status,type,ref_id,status,register_id,create_by,modify_by,gmt_create,gmt_modify) values("BACKBRACE","testpass","SLOT","xxxxxx","init","SLOT_BACKBRACE_TESTPASS" ,"zhenghai.zhang","zhenghai.zhang","%s","%s")‘ % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) cmdskilltoskillpro = ‘insert into back_brace.release_task(app_type,app_status,type,ref_id,status,register_id,create_by,modify_by,gmt_create,gmt_modify) values("SKILL","deploy","SLOT","xxxxxx","init","SLOT_SKILL_DEPLOY" ,"zhenghai.zhang","zhenghai.zhang","%s","%s")‘ % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print(cmdbacktoskill) cur.execute(cmdbacktoskill) print(cmdskilltoskillpro) cur.execute(cmdskilltoskillpro) except pymysql.Error: print("write into back_brace.release_task error!!!") cur.close() conn.commit() conn.close() def Email(to, subject, body): creds = Credentials( username=‘xxxxxx‘, password=‘xxxxxx‘) account = Account( primary_smtp_address=‘xxx@xxx.com‘, credentials=creds, autodiscover=True, access_type=DELEGATE) m = Message( account=account, subject=subject, body=HTMLBody(body), to_recipients=[Mailbox(email_address=to)]) m.send_and_save() if __name__ == ‘__main__‘: update_tvs = [] pages = 25 # 豆瓣每项电视剧只有前500部 # 美剧 英剧 韩剧 日剧 国产剧 港剧 日本动画 综艺 urlbaselist = [‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%BE%8E%E5%89%A7&sort=recommend&page_limit=20&page_start=‘, ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E8%8B%B1%E5%89%A7&sort=recommend&page_limit=20&page_start=‘, ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E9%9F%A9%E5%89%A7&sort=recommend&page_limit=20&page_start=‘, ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E6%97%A5%E5%89%A7&sort=recommend&page_limit=20&page_start=‘, ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E5%9B%BD%E4%BA%A7%E5%89%A7&sort=recommend&page_limit=20&page_start=‘, ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E6%B8%AF%E5%89%A7&sort=recommend&page_limit=20&page_start=‘, ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E6%97%A5%E6%9C%AC%E5%8A%A8%E7%94%BB&sort=recommend&page_limit=20&page_start=‘, ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%BB%BC%E8%89%BA&sort=recommend&page_limit=20&page_start=‘] for urlbase in urlbaselist: for i in range(pages): print("*"*30, i, "*"*30) tvs_list = get_tvs(urlbase, i * 20) new_tvs = tv_insert(host, user, passwd, dbme, port, table, tvs_list) for tv in new_tvs: print(tv[‘title‘],"Added") onetv = {} onetv["id"] = tv["id"] onetv["title"] = tv["title"] update_tvs.append(onetv) time.sleep(1) print(update_tvs) try: tv_new_and_sync(host, user, passwd, dbme, dbtarget, port, tabledelta, update_tvs, tablesync) # 将增加的电影写入movie_hotwords_delta表中 except: print("tv update and sync Error!") try: tv_new_to_release(host, user, passwd, dbtarget, port) except: print("tv_new_to_release error!!!") subject = ‘本次新增电视剧名称‘ body = "本次新增的电影名称为：<hr>" for movie in update_tvs: body += movie[‘title‘] + "<br>" for to in tolist: Email(to, subject, body)

 

还请各位大侠指点

Python3.5爬取豆瓣电视剧数据并且同步到mysql中

标签：app   sys   import   cts   test   strftime   amp   pass   失败   

本文系统来源：http://www.cnblogs.com/zhzhang/p/7766943.html

内容总结

以上是互联网集市为您收集整理的Python3.5爬取豆瓣电视剧数据并且同步到mysql中全部内容，希望文章能够帮你解决Python3.5爬取豆瓣电视剧数据并且同步到mysql中所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错，欢迎将互联网集市网站推荐给程序员好友。

内容备注

版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 gblab@vip.qq.com 举报，一经查实，本站将立刻删除。

内容手机端

---