scrapy-redis发送post请求
def make_request_from_data(self, data): """ 重写make_request_from_data方法,data是scrapy-redis读取redis中的[url,form_data,meta],然后发送post请求 :param data: redis中都去的请求数据,是一个list :return: 一个FormRequest对象 """ request_type = spider_conf.request_type if request_type == "post": data = json.loads(data) url = data.get(url) form_data = { } for i in data: if not i == "url": form_data[i] = data[i].strip() return FormRequest(url=url, formdata=form_data, callback=self.parse) else: url = data.decode(utf-8) return self.make_requests_from_url(url) def make_requests_from_url(self, url): """ This method is deprecated. """ warnings.warn( "Spider.make_requests_from_url method is deprecated: " "it will be removed and not be called by the default " "Spider.start_requests method in future Scrapy releases. " "Please override Spider.start_requests method instead." ) return Request(url, dont_filter=True)
make_request_from_data重写然后适当调
上一篇:
IDEA上Java项目控制台中文乱码