python爬虫之自动化爬取网页

以下为公开源码

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from bs4 import BeautifulSoup
url=https://movie.douban.com/
chrome_optins=Options()
chrome_optins.add_argument(lan=zh_CN.utf-8)
UserAgent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36
# 添加表头
chrome_optins.add_argument(User-Agent=+UserAgent)
# 开始自动化谷歌
driver=webdriver.Chrome(chrome_options=chrome_optins)
driver.maximize_window()
driver.get(url)
# 等待3秒中使网页响应成功
time.sleep(3)
# 获取网页的标题内容
print(driver.title)
# page_source 获取网页的html代码
data=driver.page_source
soup=BeautifulSoup(data,html.parser)
div_html=soup.find_all(td,class_=title)
n=0
div_htmls=soup.find(div,class_=billboard-hd).find(h2)
print(div_htmls.text)
for i in div_html:
    n=n+1
    print(1.{}.format(i.text))

首先对应自己的谷歌版本和谷歌的驱动

以下为谷歌驱动 对就谷歌版本 v2.35 v62-64 v2.34 v61-63 v2.33 v60-62 v2.32 v59-61 v2.31 v58-60 v2.30 v58-60 v2.29 v56-58 v2.28 v55-57 v2.27 v54-56 v2.26 v53-55 v2.25 v53-55 v2.24 v52-54 v2.23 v51-53 v2.22 v49-52 v2.21 v46-50 v2.20 v43-48 v2.19 v43-47 v2.18 v43-46 v2.17 v42-43 v2.13 v42-45 v2.15 v40-43 v2.14 v39-42 v2.13 v38-41 v2.12 v36-40 v2.11 v36-40 v2.10 v33-36 v2.9 v31-34 v2.8 v30-33 v2.7 v30-33 v2.6 v29-32 v2.5 v29-32 v2.4 v29-32