scrapy常用配置

# 安装 pip install fake-useragent

# middleware.py

from fake_useragent import UserAgent

class RandomUserAgentMiddlware(object):

    #随机更换 user-agent

    def __init__(self,crawler):

        super(RandomUserAgentMiddlware,self).__init__()

        self.ua = UserAgent()

    @classmethod

    def from_crawler(cls,crawler):

        return cls(crawler)

    def process_request(self,request,spider):

        request.headers.setdefault("User-Agent",self.ua.random)

# ...

# setting.py 中启动 RandomUserAgentMiddlware

DOWNLOADER_MIDDLEWARES = {

    'yourProjectName.middlewares.RandomUserAgentMiddlware': 312,

}

# 这里使用的是阿布云隧道代理

# middleware.py

import base64

# 代理服务器

proxyServer = "http://http-dyn.abuyun.com:9020"

# 代理隧道验证信息

proxyUser = "your proxyUser"

proxyPass = "your proxyPass"

proxyAuth = "Basic " + base64.urlsafe_b64encode(bytes((proxyUser + ":" + proxyPass), "ascii")).decode("utf8")

class ProxyMiddleware(object):

    def process_request(self, request, spider):

        request.meta["proxy"] = proxyServer

        request.headers["Proxy-Authorization"] = proxyAuth

# ...

# setting.py 中启动 RandomUserAgentMiddlware        

DOWNLOADER_MIDDLEWARES = {

     'yourProjectName.middlewares.ProxyMiddleware': 100,

}

# pipelines.py

from scrapy import Request

from scrapy.pipelines.images import ImagesPipeline

class ImagePipeline(ImagesPipeline):

    headers = {

		"""如果网站有headers检测就加上"""

    }

    def get_media_requests(self, item, info):

        # 循环每一张图片地址下载，若传过来的不是集合则无需循环直接 yield

        for image_url in item['imgurl']:

            headers = self.headers

            headers['Referer'] = item['from_url']

            # meta 里面的数据是从 spider 获取，然后通过 meta 传递给下面方法：file_path

            yield Request(image_url,headers=headers,meta={'foldername': item['foldername'], 'imgname': item['imgname']})

    # 重命名，若不重写这函数，图片名为哈希

    def file_path(self, request, response=None, info=None):

        pic_format = request.url.split(".")[-1]

        imgname = request.meta['imgname']

        # 接收 meta 传递过来的图集名称

        foldername = request.meta['foldername']

        # 过滤 windows 字符串

        foldername = re.sub(r'[？\\*|“<>:/]', '', foldername)

        filename = fr'{foldername}/{imgname}.{pic_format}'

        return filename

# ...

# setting.py 中启动 ImagePipeline       

DOWNLOADER_MIDDLEWARES = {

     'yourProjectName.pipelines.ImagePipeline': 200,

}

# pipelines.py

from yourProjectName.settings import MYSQL_DBNAME, MYSQL_HOST,MYSQL_PASSWORD,MYSQL_PORT,MYSQL_USER

class MysqlTwistedPipeline(object):

    """

    异步写入mysql

    """

    def __init__(self, dbpool):

        self.dbpool = dbpool

    @classmethod

    def from_settings(cls,setting):

        dbparms = dict(

            host=MYSQL_HOST,

            db=MYSQL_DBNAME,

            user=MYSQL_USER,

            passwd=MYSQL_PASSWORD,

            port=MYSQL_PORT,

            charset='utf8',

            cursorclass=pymysql.cursors.DictCursor,

            use_unicode=False,

        )

        dbpool = adbapi.ConnectionPool("pymysql", **dbparms)

        return cls(dbpool)

    def process_item(self, item, spider):

        query = self.dbpool.runInteraction(self.do_insert, item)

        query.addCallback(self.handle_error)

    def handle_error(self, failure):

        print("MysqlTwistedPipeline error is :",failure)

    def do_insert(self, cursor, item):

        insert_sql = """insert_sql"""

        cursor.execute(insert_sql, ())#有些情况需使用 pymysql.escape_string () 对 item 字段进行转义

# ...

# setting.py 中启动 ImagePipeline       

DOWNLOADER_MIDDLEWARES = {

     'yourProjectName.pipelines.MysqlTwistedPipeline': 200,

}