python实现的一个火车票转让信息采集器
内容导读
互联网集市收集整理的这篇技术教程文章主要介绍了python实现的一个火车票转让信息采集器,小编现在分享给大家,供广大互联网技能从业者学习和参考。文章包含3007字,纯文字阅读大概需要5分钟。
内容图文
![python实现的一个火车票转让信息采集器](/upload/InfoBanner/zyjiaocheng/436/830b63ecb50d4676a1972d3bb7b750e0.jpg)
#coding: utf-8 ''' 春运查询火车票转让信息 Author: piglei2007@gmail.com Date: 2011.01.25 ''' import re import os import time import urlparse import datetime import traceback import urllib2 import socket socket.setdefaulttimeout(20) BLANK_RE = re.compile(r"\s+") opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) opener.addheaders = [ ("User-agent", "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.9.1) Gecko/20090704 Firefox/3.5"), ("Accept", "*/*"), ] urllib2.install_opener(opener) from BeautifulSoup import BeautifulSoup SOURCE = { "58": "http://bj.58.com/huochepiao/?Num=%(train)s&StartTime=%(date)s00", "ganji": "http://bj.ganji.com/piao/cc_%(train)s/%(date)s/", } RECORD_FILE = "/tmp/ticket_records.txt" def parse_record(): try: return set([x.strip() for x in open(RECORD_FILE, "r").readlines()]) except IOError: open(RECORD_FILE, "w") return set() def flush_record(records): open(RECORD_FILE, "w").write("\n".join(records)) def main(config): """ 开始抓取 """ existed = parse_record() to_email = [] for train in config["trains"]: for date in config["dates"]: for type, _url in SOURCE.items(): url = _url % dict(train=train, date=date) content = urllib2.urlopen(url).read() soup = BeautifulSoup(content) result = parse_content(type, soup, train) for url, text in result: url = urlparse.urljoin(_url, url) # 只要卧铺! if url not in existed and u"卧" in text: to_email.append([text, url]) existed.add(url) if to_email: content = "".join( [x for x in [" | ".join(y) for y in to_email]] ).encode("utf-8") simple_mail(config["people"], content) flush_record(existed) def parse_content(type, soup, train): """ 获得车次信息 """ result = [] if type == "58": info_table = soup.find("table", id="infolist") if info_table: for x in info_table.findAll("tr", text=re.compile(ur"%s(?!时刻表)" % train, re.I)): a = x.parent _text = BLANK_RE.sub("", a.text) result.append([a["href"], _text]) if type == "ganji": for x in soup.findAll("dl", {"class": "list_piao"}): a = x.dt.a result.append([a["href"], a.text]) return result EMAIL_HOST = 'smtp.sohu.com' EMAIL_HOST_USER = 'yourname@sohu.com' EMAIL_HOST_PASSWORD = 'yourpassword' EMAIL_PORT = 25 def simple_mail(to, content): """ 发送邮件 """ import smtplib from email.mime.text import MIMEText msgRoot = MIMEText(content, 'html', 'UTF-8') msgRoot['Subject'] = "[%s]有票来啦!!!!" % datetime.datetime.today().isoformat(" ") msgRoot['From'] = EMAIL_HOST_USER msgRoot['To'] = ", ".join(to) s = smtplib.SMTP(EMAIL_HOST, EMAIL_PORT) s.login(EMAIL_HOST_USER, EMAIL_HOST_PASSWORD) s.sendmail(EMAIL_HOST_USER, to, msgRoot.as_string()) s.close() def switch_time_zone(): """ 切换时区 """ os.environ["TZ"] = "Asia/Shanghai" time.tzset() switch_time_zone() if __name__ == '__main__': config = { "trains": ("k471",), "dates": ("20110129",), "people": ( "youremail@sohu.com", ) } try: main(config) print "%s: ok" % datetime.datetime.today() except Exception, e: print traceback.format_exc()
然后放入cron,你懂的。
内容总结
以上是互联网集市为您收集整理的python实现的一个火车票转让信息采集器全部内容,希望文章能够帮你解决python实现的一个火车票转让信息采集器所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错,欢迎将互联网集市网站推荐给程序员好友。
内容备注
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 gblab@vip.qq.com 举报,一经查实,本站将立刻删除。
内容手机端
扫描二维码推送至手机访问。