Python 爬虫（六）：Scrapy 爬取景区信息

def request_seen(self, request):
    # 计算 request 的指纹
    fp = self.request_fingerprint(request)
    # 判断指纹是否已经存在
    if fp in self.fingerprints:
        # 存在
        return True
    # 不存在，加入到指纹集合中
    self.fingerprints.add(fp)
    if self.file:
        self.file.write(fp + os.linesep)

def request_fingerprint(request, include_headers=None):
    if include_headers:
        include_headers = tuple(to_bytes(h.lower())
                                 for h in sorted(include_headers))
    cache = _fingerprint_cache.setdefault(request, {})
    if include_headers not in cache:
        fp = hashlib.sha1()
        fp.update(to_bytes(request.method))
        fp.update(to_bytes(canonicalize_url(request.url)))
        fp.update(request.body or b'')
        if include_headers:
            for hdr in include_headers:
                if hdr in request.headers:
                    fp.update(hdr)
                    for v in request.headers.getlist(hdr):
                        fp.update(v)
        cache[include_headers] = fp.hexdigest()
    return cache[include_headers]

fp = hashlib.sha1()
...
cache[include_headers] = fp.hexdigest()

def __init__(self, path=None, debug=False):
	self.file = None
	self.fingerprints = set()
	self.logdupes = True
	self.debug = debug
	self.logger = logging.getLogger(__name__)
	if path:
		self.file = open(os.path.join(path, 'requests.seen'), 'a+')
		self.file.seek(0)
		self.fingerprints.update(x.rstrip() for x in self.file)

import scrapy
class TicketspiderItem(scrapy.Item):

area = scrapy.Field()

sight = scrapy.Field()

level = scrapy.Field()

price = scrapy.Field()

pass

import scrapy

from ticketSpider.items import TicketspiderItem

class QunarSpider(scrapy.Spider):

name = 'qunar'

allowed_domains = ['piao.qunar.com']

start_urls = ['https://piao.qunar.com/ticket/list.htm?keyword=%E5%8C%97%E4%BA%AC®ion=&from=mpl_search_suggest']
def parse(self, response):
    sight_items = response.css('#search-list .sight_item')
    for sight_item in sight_items:
        item = TicketspiderItem()
        item['area'] = sight_item.css('::attr(data-districts)').extract_first()
        item['sight'] = sight_item.css('::attr(data-sight-name)').extract_first()
        item['level'] = sight_item.css('.level::text').extract_first()
        item['price'] = sight_item.css('.sight_item_price em::text').extract_first()
        yield item
    # 翻页
    next_url = response.css('.next::attr(href)').extract_first()
    if next_url:
        next_url = "https://piao.qunar.com" + next_url
        yield scrapy.Request(
            next_url,
            callback=self.parse
        )</code></pre></div></div><p>简单介绍一下：</p><ul class="ul-level-0"><li>name：爬虫名</li><li>allowed_domains：允许爬取的域名</li><li>atart_urls：爬取网站初始请求的 url（可定义多个）</li><li>parse 方法：解析网页的方法</li><li>response 参数：请求网页后返回的内容</li></ul><p><strong>yield</strong></p><p>在上面的代码中我们看到有个 <code>yield</code>，简单说一下，<code>yield</code> 是一个关键字，作用和 <code>return</code> 差不多，差别在于 <code>yield</code> 返回的是一个生成器（在 Python 中，一边循环一边计算的机制，称为生成器），它的作用是：有利于减小服务器资源，在列表中所有数据存入内存，而生成器相当于一种方法而不是具体的信息，占用内存小。</p><p><strong>爬虫伪装</strong></p><p>通常需要对爬虫进行一些伪装，我们也简单处理一下，介绍一个最简单的方法：</p><ul class="ul-level-0"><li>使用终端命令 <code>pip install scrapy-fake-useragent</code> 安装</li><li>在 settings.py 文件中添加如下代码：</li></ul><div class="rno-markdown-code"><div class="rno-markdown-code-toolbar"><div class="rno-markdown-code-toolbar-info"><div class="rno-markdown-code-toolbar-item is-type"><span class="is-m-hidden">代码语言：</span>javascript</div></div><div class="rno-markdown-code-toolbar-opt"><div class="rno-markdown-code-toolbar-copy"><i class="icon-copy"></i><span class="is-m-hidden">复制</span></div></div></div><div class="developer-code-block"><pre class="prism-token token line-numbers language-javascript"><code class="language-javascript" style="margin-left:0">DOWNLOADER_MIDDLEWARES = {
# 关闭默认方法
&#39;scrapy.downloadermiddlewares.useragent.UserAgentMiddleware&#39;: None, 
# 开启
&#39;scrapy_fake_useragent.middleware.RandomUserAgentMiddleware&#39;: 400, 

}

import csv

class TicketspiderPipeline(object):

def init(self):

self.f = open('ticker.csv', 'w', encoding='utf-8', newline='')

self.fieldnames = ['area', 'sight', 'level', 'price']

self.writer = csv.DictWriter(self.f, fieldnames=self.fieldnames)

self.writer.writeheader()

def process_item(self, item, spider):

self.writer.writerow(item)

return item
def close(self, spider):
    self.f.close()</code></pre></div></div><p>然后，将 settings.py 文件中如下代码：</p><div class="rno-markdown-code"><div class="rno-markdown-code-toolbar"><div class="rno-markdown-code-toolbar-info"><div class="rno-markdown-code-toolbar-item is-type"><span class="is-m-hidden">代码语言：</span>javascript</div></div><div class="rno-markdown-code-toolbar-opt"><div class="rno-markdown-code-toolbar-copy"><i class="icon-copy"></i><span class="is-m-hidden">复制</span></div></div></div><div class="developer-code-block"><pre class="prism-token token line-numbers language-javascript"><code class="language-javascript" style="margin-left:0">ITEM_PIPELINES = {
&#39;ticketSpider.pipelines.TicketspiderPipeline&#39;: 300,

}

from scrapy.cmdline import execute

execute('scrapy crawl 爬虫名'.split())