'''
爬取扇贝网python必背词汇表
接口地址：https://www.shanbay.com/wordlist/110521/232414/
要求：获取所有的python词汇数据，形成字典，然后存储数据
思路：
第一页：https://www.shanbay.com/wordlist/110521/232414/
第二页：https://www.shanbay.com/wordlist/110521/232414/?page=2
'''
import requests,re,os,json
from lxml import etree
class Shanbei:
def __call__(self, *args, **kwargs):
self.get_xml(*args)
def get_xml(self,pages):
'''
获取扇贝英语python页 内容 转换为xml
:return:获取扇贝英语python页的xml
'''
base_url = 'https://www.shanbay.com/wordlist/110521/232414/?page='
for page in range(1,pages+1):
base_html = requests.get('https://www.shanbay.com/wordlist/110521/232414/?page=1').text
base_xml = etree.HTML(requests.get(base_url+str(page)).text)
pages = page
# print(base_xml)
with open('贝壳应英语.html','w',encoding='utf-8') as f :
f.write(base_html)
self.get_python(base_html,base_xml,pages)
def get_python(self,base_html,base_xml,pages):
'''
处理python页面数据
:param base_xml:
:return:
'''
#获取标题
# main_title = base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][1]/div[@class='span8']/div[@class='well']/div[@class='row']/div[@class='span6']/h4/text()")
#胆子获取
worlds_rule = re.compile(r'<h4>(.*?)</h4>',re.S)
main_title = worlds_rule.findall(base_html)
get_words = re.compile(r'[a-z][A-Z]+',re.I)
world_title = (get_words.findall(main_title[0]))[0]
# print(base_html)
print(world_title)
# #获取本页词条数量
trs_length = len(base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][2]/div[@class='span8']/table[@class='table table-bordered table-striped']/tbody/tr[@class='row']"))
print(trs_length)
#准备大字典
shell_word_dict = {
f'必备单词{world_title}':{},
}
#根据词条数量获取循环次数
for line in range(int(trs_length)):
word = base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][2]/div[@class='span8']/table[@class='table table-bordered table-striped']/tbody/tr[@class='row'][{}]/td[@class='span2']/strong/text()".format(line+1))
word_description = base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][2]/div[@class='span8']/table[@class='table table-bordered table-striped']/tbody/tr[@class='row'][{}]/td[@class='span10']/text()".format(line+1))
# print(word,word_description)
shell_word_dict.get(f'必备单词{world_title}')[word[0]] = word_description[0]
print(shell_word_dict)
#储存文件 注意：此处保存的是循环外的字典
path = './扇贝单词/'+'第'+str(pages)+'页' +'/'
print(path)
if not os.path.exists(path):
os.makedirs(path)
file_path = path+'第'+str(pages)+'页'
with open(file_path,'w',encoding='utf-8')as f:
f.write(json.dumps(shell_word_dict))
if __name__ == '__main__':
# pages = int(input('请输入需要爬取的页数'))
pages = 3
shanbei = Shanbei()
shanbei(pages)
