scrapy-redis发送post请求
def make_request_from_data(self, data):
"""
重写make_request_from_data方法,data是scrapy-redis读取redis中的[url,form_data,meta],然后发送post请求
:param data: redis中都去的请求数据,是一个list
:return: 一个FormRequest对象
"""
request_type = spider_conf.request_type
if request_type == "post":
data = json.loads(data)
url = data.get(url)
form_data = {
}
for i in data:
if not i == "url":
form_data[i] = data[i].strip()
return FormRequest(url=url, formdata=form_data, callback=self.parse)
else:
url = data.decode(utf-8)
return self.make_requests_from_url(url)
def make_requests_from_url(self, url):
""" This method is deprecated. """
warnings.warn(
"Spider.make_requests_from_url method is deprecated: "
"it will be removed and not be called by the default "
"Spider.start_requests method in future Scrapy releases. "
"Please override Spider.start_requests method instead."
)
return Request(url, dont_filter=True)
make_request_from_data重写然后适当调
上一篇:
IDEA上Java项目控制台中文乱码
