import requests as reqs
import threading
import time
#Some User Agents
hds={'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6',
'User-Agent':'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11',
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'}
from scrapy import Selector
def html_parse(html_str,xpath_expr):
sel = Selector(text=htmltext)
xp = lambda x: sel.xpath(x).extract()
return xp(xpathstr)
class myThread_detail(threading.Thread):
"""docstring for myThead"""
def __init__(self, url,people_nums):
super(myThread_detail, self).__init__()
self.url = url
self.people_nums=people_nums
def run(self,):
loc=threading.Lock()
data=reqs.get(self.url,headers=hds).content.decode()
#返回评论人数
loc.acquire()
try:
self.people_nums.append(html_parse(data,'//a[@class="rating_people"]//span/text()')[0])
except IndexError as e:
self.people_nums.append(" ")
loc.release()
# 根据标签爬取单个标签下的图书列表
def book_spider(book_tag):
#url='http://www.douban.com/tag/小说/book?start=0' # For Test
page_num=0
url=r'http://www.douban.com/tag/'+book_tag+'/book?start='
for i in range(0,15*10,15):
urls=url+str(i)
print(urls)
data=reqs.get(urls,headers=hds).content.decode()
# ==============以下是要爬取的三个字段，要爬取更多的字段可以在这里扩展================
# 书名
booknames =html_parse(data,'//a[@class="title"]/text()')
#描述
descs =html_parse(data,'//div[@class="desc"]/text()')
detail_links =html_parse(data,'''//a[@class="title"]/@href''')
# 评论人数
people_nums=[]
#进入详情页面抓取，评论数
tt=[]
for link in detail_links:
t=myThread_detail(link,people_nums)
t.start()
time.sleep(0.1)
tt.append(t)
for th in tt:
th.join()
yield (booknames,descs,people_nums)
#获取评论人数，获取详细信息
# https://book.douban.com/subject/6082808/?from=tag_all for test
def get_people_num(url):
data=reqs.get(url,headers=hds).content.decode()
#返回评论人数
return html_parse(data,'//a[@class="rating_people"]//span/text()')
#爬取多个标签（需要传入标签列表参数）
def do_spider(book_tag_lists):
for i in book_tag_lists:
for item in book_spider(i):
# 把搞到的数据打印出来
print(item)
# 返回的是个复数形式，也就是说是个多个标签的最终结果
# return book_lists
return 0
#把数据写到电子表格中。
def print_book_lists_excel(book_lists,book_tag_lists):
'''每个标签一张表，每张表就一个标签的书籍列表;
如果你有兴趣自己写吧，也可以问我！'''
pass
if __name__=='__main__':
#book_tag_lists = ['心理','判断与决策','算法','数据结构','经济','历史']
#book_tag_lists = ['传记','哲学','编程','创业','理财','社会学','佛教']
#book_tag_lists = ['思想','科技','科学','web','股票','爱情','两性']
#book_tag_lists = ['计算机','机器学习','linux','android','数据库','互联网']
#book_tag_lists = ['数学']
#book_tag_lists = ['摄影','设计','音乐','旅行','教育','成长','情感','育儿','健康','养生']
#book_tag_lists = ['商业','理财','管理']
#book_tag_lists = ['名著']
#book_tag_lists = ['科普','经典','生活','心灵','文学']
#book_tag_lists = ['科幻','思维','金融']
# book_tag_lists = ['小说','时间管理','投资','文化','宗教']
book_tag_lists = ['小说']
book_lists=do_spider(book_tag_lists)
print_book_lists_excel(book_lists,book_tag_lists)
###########################################################################
# 运行一次就好，若果你运行第二次，立马会被封ip ，当然你可以换一个ip；继续运行。 ##
参考地址：
https://github.com/lanbing510/DouBanSpider/blob/master/doubanSpider.py
