【Python笔记】爬虫连续多页爬取数据

爬虫四部曲：

1、获取网页；

2、解析网页文本；

3、提取需要的信息；

4、存储信息。

import requests
from lxml import etree
import xlsxwriter
url = 'https://...'
header = {'user-agent': '...'}
flag = True
workbook = xlsxwriter.Workbook('taoche_spider.xlsx')
worksheet = workbook.add_worksheet()
headings = ['**','**','**']
# worksheet.write(0,0,'**')
# worksheet.write(0, 1, '**')
# worksheet.write(0, 2, '**')
worksheet.write_row('A1',headings)
k = 1
while flag:
    html = requests.get(url, headers=header).text
    list = etree.HTML(html)
    lis = list.xpath('//ul[@class="gongge_ul"]/li')
    for selector in lis:
        name = selector.xpath('//div[@class=