摘要：百度云搜索，搜各種資料搜網(wǎng)盤，搜各種資料編寫(xiě)爬蟲(chóng)文件循環(huán)抓取內(nèi)容方法，將指定的地址添加到下載器下載頁(yè)面，兩個(gè)必須參數(shù)，參數(shù)頁(yè)面處理函數(shù)使用時(shí)需要方法，是庫(kù)下的方法，是自動(dòng)拼接，如果第二個(gè)參數(shù)的地址是相對(duì)路徑會(huì)自動(dòng)與第一個(gè)參數(shù)拼接導(dǎo)

編寫(xiě)spiders爬蟲(chóng)文件循環(huán)抓取內(nèi)容

Request()方法，將指定的url地址添加到下載器下載頁(yè)面，兩個(gè)必須參數(shù)，
　　參數(shù)：
　　url="url"
　　callback=頁(yè)面處理函數(shù)
　　使用時(shí)需要yield Request()

parse.urljoin()方法，是urllib庫(kù)下的方法，是自動(dòng)url拼接，如果第二個(gè)參數(shù)的url地址是相對(duì)路徑會(huì)自動(dòng)與第一個(gè)參數(shù)拼接

#?-*-?coding:?utf-8?-*-
import?scrapy
from?scrapy.http?import?Request?????????????????????????????#導(dǎo)入url返回給下載器的方法
from?urllib?import?parse????????????????????????????????????#導(dǎo)入urllib庫(kù)里的parse模塊

class?PachSpider(scrapy.Spider):
????name?=?"pach"
????allowed_domains?=?["blog.jobbole.com"]??????????????????#起始域名
????start_urls?=?["http://blog.jobbole.com/all-posts/"]?????#起始url

????def?parse(self,?response):
????????"""
????????獲取列表頁(yè)的文章url地址，交給下載器
????????"""
????????#獲取當(dāng)前頁(yè)文章url
????????lb_url?=?response.xpath("http://a[@class="archive-title"]/@href").extract()??#獲取文章列表url
????????for?i?in?lb_url:
????????????#?print(parse.urljoin(response.url,i))?????????????????????????????????????????????#urllib庫(kù)里的parse模塊的urljoin()方法，是自動(dòng)url拼接，如果第二個(gè)參數(shù)的url地址是相對(duì)路徑會(huì)自動(dòng)與第一個(gè)參數(shù)拼接
????????????yield?Request(url=parse.urljoin(response.url,?i),?callback=self.parse_wzhang)??????#將循環(huán)到的文章url添加給下載器，下載后交給parse_wzhang回調(diào)函數(shù)

????????#獲取下一頁(yè)列表url,交給下載器，返回給parse函數(shù)循環(huán)
????????x_lb_url?=?response.xpath("http://a[@class="next?page-numbers"]/@href").extract()?????????#獲取下一頁(yè)文章列表url
????????if?x_lb_url:
????????????yield?Request(url=parse.urljoin(response.url,?x_lb_url[0]),?callback=self.parse)?????#獲取到下一頁(yè)url返回給下載器，回調(diào)給parse函數(shù)循環(huán)進(jìn)行

????def?parse_wzhang(self,response):
????????title?=?response.xpath("http://div[@class="entry-header"]/h1/text()").extract()???????????#獲取文章標(biāo)題
????????print(title)

Request()函數(shù)在返回url時(shí)，同時(shí)可以通過(guò)meta屬性返回一個(gè)自定義字典給回調(diào)函數(shù)

#?-*-?coding:?utf-8?-*-
import?scrapy
from?scrapy.http?import?Request?????????????????????????????#導(dǎo)入url返回給下載器的方法
from?urllib?import?parse????????????????????????????????????#導(dǎo)入urllib庫(kù)里的parse模塊
from?adc.items?import?AdcItem???????????????????????????????#導(dǎo)入items數(shù)據(jù)接收模塊的接收類

class?PachSpider(scrapy.Spider):
????name?=?"pach"
????allowed_domains?=?["blog.jobbole.com"]??????????????????#起始域名
????start_urls?=?["http://blog.jobbole.com/all-posts/"]?????#起始url

????def?parse(self,?response):
????????"""
????????獲取列表頁(yè)的文章url地址，交給下載器
????????"""
????????#獲取當(dāng)前頁(yè)文章url
????????lb?=?response.css("div?.post.floated-thumb")??#獲取文章列表區(qū)塊,css選擇器
????????#?print(lb)
????????for?i?in?lb:
????????????lb_url?=?i.css(".archive-title?::attr(href)").extract_first("")?????#獲取區(qū)塊里文章url
????????????#?print(lb_url)

????????????lb_img?=?i.css(".post-thumb?img?::attr(src)").extract_first("")?????#獲取區(qū)塊里文章縮略圖
????????????#?print(lb_img)

????????????yield?Request(url=parse.urljoin(response.url,?lb_url),?meta={"lb_img":parse.urljoin(response.url,?lb_img)},?callback=self.parse_wzhang)??????#將循環(huán)到的文章url添加給下載器，下載后交給parse_wzhang回調(diào)函數(shù)

????????#獲取下一頁(yè)列表url,交給下載器，返回給parse函數(shù)循環(huán)
????????x_lb_url?=?response.css(".next.page-numbers?::attr(href)").extract_first("")?????????#獲取下一頁(yè)文章列表url
????????if?x_lb_url:
????????????yield?Request(url=parse.urljoin(response.url,?x_lb_url),?callback=self.parse)?????#獲取到下一頁(yè)url返回給下載器，回調(diào)給parse函數(shù)循環(huán)進(jìn)行

????def?parse_wzhang(self,response):
????????title?=?response.css(".entry-header?h1?::text").extract()???????????#獲取文章標(biāo)題
????????#?print(title)

????????tp_img?=?response.meta.get("lb_img",?"")????????????????????????????#接收meta傳過(guò)來(lái)的值，用get獲取防止出錯(cuò)
????????#?print(tp_img)

????????shjjsh?=?AdcItem()???????????????????????????????????????????????????????????????????#實(shí)例化數(shù)據(jù)接收類
????????shjjsh["title"]?=?title??????????????????????????????????????????????????????????????#將數(shù)據(jù)傳輸給items接收模塊的指定類
????????shjjsh["img"]?=?tp_img

????????yield?shjjsh????????????????????????????????#將接收對(duì)象返回給pipelines.py處理模塊

*

Scrapy內(nèi)置圖片下載器使用

Scrapy給我們內(nèi)置了一個(gè)圖片下載器在crapy.pipelines.images.ImagesPipeline，專門用于將爬蟲(chóng)抓取到圖片url后將圖片下載到本地

第一步、爬蟲(chóng)抓取圖片URL地址后，填充到?items.py文件的容器函數(shù)

　　爬蟲(chóng)文件

#?-*-?coding:?utf-8?-*-
import?scrapy
from?scrapy.http?import?Request?????????????????????????????#導(dǎo)入url返回給下載器的方法
from?urllib?import?parse????????????????????????????????????#導(dǎo)入urllib庫(kù)里的parse模塊
from?adc.items?import?AdcItem???????????????????????????????#導(dǎo)入items數(shù)據(jù)接收模塊的接收類

class?PachSpider(scrapy.Spider):
????name?=?"pach"
????allowed_domains?=?["blog.jobbole.com"]??????????????????#起始域名
????start_urls?=?["http://blog.jobbole.com/all-posts/"]?????#起始url

????def?parse(self,?response):
????????"""
????????獲取列表頁(yè)的文章url地址，交給下載器
????????"""
????????#獲取當(dāng)前頁(yè)文章url
????????lb?=?response.css("div?.post.floated-thumb")??#獲取文章列表區(qū)塊,css選擇器
????????#?print(lb)
????????for?i?in?lb:
????????????lb_url?=?i.css(".archive-title?::attr(href)").extract_first("")?????#獲取區(qū)塊里文章url
????????????#?print(lb_url)

????????????lb_img?=?i.css(".post-thumb?img?::attr(src)").extract_first("")?????#獲取區(qū)塊里文章縮略圖
????????????#?print(lb_img)

????????????yield?Request(url=parse.urljoin(response.url,?lb_url),?meta={"lb_img":parse.urljoin(response.url,?lb_img)},?callback=self.parse_wzhang)??????#將循環(huán)到的文章url添加給下載器，下載后交給parse_wzhang回調(diào)函數(shù)

????????#獲取下一頁(yè)列表url,交給下載器，返回給parse函數(shù)循環(huán)
????????x_lb_url?=?response.css(".next.page-numbers?::attr(href)").extract_first("")?????????#獲取下一頁(yè)文章列表url
????????if?x_lb_url:
????????????yield?Request(url=parse.urljoin(response.url,?x_lb_url),?callback=self.parse)?????#獲取到下一頁(yè)url返回給下載器，回調(diào)給parse函數(shù)循環(huán)進(jìn)行

????def?parse_wzhang(self,response):
????????title?=?response.css(".entry-header?h1?::text").extract()???????????#獲取文章標(biāo)題
????????#?print(title)

????????tp_img?=?response.meta.get("lb_img",?"")????????????????????????????#接收meta傳過(guò)來(lái)的值，用get獲取防止出錯(cuò)
????????#?print(tp_img)

????????shjjsh?=?AdcItem()???????????????????????????????????????????????????????????????????#實(shí)例化數(shù)據(jù)接收類
????????shjjsh["title"]?=?title??????????????????????????????????????????????????????????????#將數(shù)據(jù)傳輸給items接收模塊的指定類
????????shjjsh["img"]?=?[tp_img]

????????yield?shjjsh????????????????????????????????#將接收對(duì)象返回給pipelines.py處理模塊

第二步、設(shè)置?items.py?文件的容器函數(shù)，接收爬蟲(chóng)獲取到的數(shù)據(jù)填充

#?-*-?coding:?utf-8?-*-

#?Define?here?the?models?for?your?scraped?items
#
#?See?documentation?in:
#?http://doc.scrapy.org/en/latest/topics/items.html

import?scrapy

#items.py,文件是專門用于，接收爬蟲(chóng)獲取到的數(shù)據(jù)信息的，就相當(dāng)于是容器文件

class?AdcItem(scrapy.Item):????#設(shè)置爬蟲(chóng)獲取到的信息容器類
????title?=?scrapy.Field()?????#接收爬蟲(chóng)獲取到的title信息
????img?=?scrapy.Field()???????#接收縮略圖
????img_tplj?=?scrapy.Field()??#圖片保存路徑

第三步、在pipelines.py使用crapy內(nèi)置的圖片下載器

1、首先引入內(nèi)置圖片下載器

2、自定義一個(gè)圖片下載內(nèi)，繼承crapy內(nèi)置的ImagesPipeline圖片下載器類

3、使用ImagesPipeline類里的item_completed()方法獲取到圖片下載后的保存路徑

4、在settings.py設(shè)置文件里，注冊(cè)自定義圖片下載器類，和設(shè)置圖片保存路徑

#?-*-?coding:?utf-8?-*-

#?Define?your?item?pipelines?here
#
#?Don"t?forget?to?add?your?pipeline?to?the?ITEM_PIPELINES?setting
#?See:?http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from?scrapy.pipelines.images?import?ImagesPipeline??#導(dǎo)入圖片下載器模塊

class?AdcPipeline(object):??????????????????????#定義數(shù)據(jù)處理類，必須繼承object
????def?process_item(self,?item,?spider):???????#process_item(item)為數(shù)據(jù)處理函數(shù)，接收一個(gè)item，item里就是爬蟲(chóng)最后yield?item?來(lái)的數(shù)據(jù)對(duì)象
????????print("文章標(biāo)題是："?+?item["title"][0])
????????print("文章縮略圖url是："?+?item["img"][0])
????????print("文章縮略圖保存路徑是："?+?item["img_tplj"])??#接收?qǐng)D片下載器填充的，圖片下載后的路徑

????????return?item

class?imgPipeline(ImagesPipeline):??????????????????????#自定義一個(gè)圖片下載內(nèi)，繼承crapy內(nèi)置的ImagesPipeline圖片下載器類
????def?item_completed(self,?results,?item,?info):??????#使用ImagesPipeline類里的item_completed()方法獲取到圖片下載后的保存路徑
????????for?ok,?value?in?results:
????????????img_lj?=?value["path"]?????#接收?qǐng)D片保存路徑
????????????#?print(ok)
????????????item["img_tplj"]?=?img_lj??#將圖片保存路徑填充到items.py里的字段里
????????return?item????????????????????#將item給items.py?文件的容器函數(shù)

????#注意：自定義圖片下載器設(shè)置好后，需要在

在settings.py設(shè)置文件里，注冊(cè)自定義圖片下載器類，和設(shè)置圖片保存路徑

IMAGES_URLS_FIELD?設(shè)置要下載圖片的url地址，一般設(shè)置的items.py里接收的字段
IMAGES_STORE?設(shè)置圖片保存路徑

#?Configure?item?pipelines
#?See?http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES?=?{
???"adc.pipelines.AdcPipeline":?300,??#注冊(cè)adc.pipelines.AdcPipeline類，后面一個(gè)數(shù)字參數(shù)表示執(zhí)行等級(jí)，
???"adc.pipelines.imgPipeline":?1,????#注冊(cè)自定義圖片下載器,數(shù)值越小，越優(yōu)先執(zhí)行
}

IMAGES_URLS_FIELD?=?"img"?????????????????????????????#設(shè)置要下載圖片的url字段，就是圖片在items.py里的字段里
lujin?=?os.path.abspath(os.path.dirname(__file__))
IMAGES_STORE?=?os.path.join(lujin,?"img")?????????????#設(shè)置圖片保存路徑

【轉(zhuǎn)載自：http://www.lqkweb.com】