爬虫---获取网页数据的几种方式

1.post方式（有道词典）

# -*- coding:utf-8 -*-

import urllib
import urllib2

url = http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule
headers = {
User-agent:Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36
}


key = raw_input(请输入翻译的英语:)

formdata = {
i:hello,
from:AUTO,
to:AUTO,
smartresult:dict,
client:fanyideskweb,
salt:1526387112925,
doctype:json,
version:2.1,
keyfrom:fanyi.web,
action:FY_BY_CLICKBUTTION,
typoResult:false,
}

data = urllib.urlencode(formdata)

request = urllib2.Request(url ,data=data, headers=headers)

print(urllib2.urlopen(request).read())

2.ajax方式(豆瓣电影网)

# -*- coding:utf-8 -*-

import urllib
import urllib2

url = https://movie.douban.com/j/new_search_subjects?

headers = {User-agent:Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36}

formdata = {
    sort:T,
    range:0,10,
    tags:,
    start:10,
    genres:科幻,
}
data = urllib.urlencode(formdata)

request = urllib2.Request(url, data=data, headers=headers)

print(urllib2.urlopen(request).read())

3.cookie登录方式()

# -*- coding:utf-8 -*-

import urllib2

url = http://blog..net/jian15093532273/article/details/80313793
header = {
    Accept:*/*,
    Accept-Language:zh-CN,zh;q=0.9,
    Connection:keep-alive,
    Content-Length:286,
    Content-Type:text/plain;charset=UTF-8,
    Cookie:uuid_tt_dd=5624585880944396552_20180509; kd_user_id=7059fcfd-6b66-454b-bb99-f87b70676dde; UN=jian15093532273; dc_session_id=10_1526392746369.315469; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1526284068,1526381959,1526388494,1526392747; smidV2=20180515215924c4d5f3ee6537ace030bddb96f86f76020096344313ac34450; kd_0e1a1f29-37da-4c44-8a33-b4735dc85f10_kuickDeal_pageIndex=0; kd_0e1a1f29-37da-4c44-8a33-b4735dc85f10_kuickDeal_leaveTime=1526392783862; UserName=jian15093532273; UserInfo=OYXG1lKtKnuQ5RuGXlV0ddyWw4m%2FHJnX%2B%2Ftpib66T3S%2FjZ7%2Bv%2BuBm6yb90pmiyaRMXpMvJrbQ31cvjx%2Fu1Lp1VRW3NTSkn53a5C4ZF7aPED%2Fm%2F0dM%2BCekDkMqEF2rgU2a5R5Annds%2B5S4sEnaKk%2FCA%3D%3D; UserNick=OnlyLove_%E7%90%B3; AU=315; BT=1526392781724; dc_tos=p8rw7l; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1526392786,
    Host:re..net,
    Origin:https://www..net,
    Referer:https://www..net/,
    User-Agent:Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36,
}

request = urllib2.Request(url,  headers=header)

print(urllib2.urlopen(request).read())

3.关闭ssl安全证书验证(12306)

# -*- coding:utf-8 -*-

import urllib2
import ssl

# 忽略ssl安全认证
context = ssl._create_unverified_context()

url = https://www.12306.cn/mormhweb/

request = urllib2.Request(url)

response = urllib2.urlopen(request, context=context)

print(response.read())