scrapy-redis发送post请求

def make_request_from_data(self, data):
        """
        重写make_request_from_data方法,data是scrapy-redis读取redis中的[url,form_data,meta],然后发送post请求
        :param data: redis中都去的请求数据,是一个list
        :return: 一个FormRequest对象
      """
        request_type = spider_conf.request_type
        if request_type == "post":
            data = json.loads(data)
            url = data.get(url)
            form_data = {
          
   }
            for i in data:
                if not i == "url":
                    form_data[i] = data[i].strip()
            return FormRequest(url=url, formdata=form_data, callback=self.parse)
        else:
            url = data.decode(utf-8)
            return self.make_requests_from_url(url)
   def make_requests_from_url(self, url):
     """ This method is deprecated. """
     warnings.warn(
         "Spider.make_requests_from_url method is deprecated: "
         "it will be removed and not be called by the default "
         "Spider.start_requests method in future Scrapy releases. "
         "Please override Spider.start_requests method instead."
     )
     return Request(url, dont_filter=True)

make_request_from_data重写然后适当调

经验分享 程序员 微信小程序 职场和发展