将python中爬取的数据保存到数据库中

要用python关联爬取数据，保存到数据库中

分两步，第一步就是爬取数据，第二步才是保存到数据库

爬取数据

需要的头文件有：

# -*- coding:UTF-8 -*-
import pymysql as mdb
import pymysql.cursors
import requests
from bs4 import BeautifulSoup
import bs4

解析网址

def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""

爬取数据

def fillnameList(ulist1, html):
    soup = BeautifulSoup(html, "html.parser")

    a = soup.find(div, class_=hcent_m1j_2n).findAll("dt")
    for i in a:
        tds = i("a")
        ulist1.append([tds[0].string])

def fillUnivList(ulist, html):
    soup = BeautifulSoup(html, "html.parser")

    a = soup.find(table, class_=text12).findAll("tr")
    for i in a:
        tds = i("td")
        ulist.append([tds[0].string, tds[1].string])
        # ulist.append([tds[1].string])

将爬取的数据插入到mysql数据库

def printUnivList(ulist, num):
    # h=[]
    tplt = "{0:^10}	{1:^10}"#0表示位置，^表示居中，10表示占10个位置
    for i in range(num):
        if (i == 7) or (i == 0) or (i == 1):
            u = ulist[I]
            print(tplt.format(u[0].encode("utf-8"), u[1].encode("utf-8"),chr(255)))
    q = ulist[0]
    w = ulist[1]
    e = ulist[7]
    con = mdb.connect(localhost, root, zhanghang, nongyao);
    with con:
        cur = con.cursor()
        cur.execute(
            "INSERT INTO NongYao(nyname,cname,applications) VALUES(%s,%s,%s)" % (tplt.format(q[0].encode("utf-8"),
                                                                                            q[1].encode("utf-8"),
                                                                                                   chr(255)),
                                                                                       tplt.format(w[0].encode("utf-8"),
                                                                                                   w[1].encode("utf-8"),
                                                                                                   chr(255)),
                                                                                       tplt.format(e[0].encode("utf-8"),
                                                                                                   e[1].encode("utf-8"),
                                                                                                   chr(255))))

执行

def work(ulist1, url_1, num):
    tplt = "{0:^10}"
    for i in range(num):
        # if (i==7)or(i==0)or(i==1)or(i==2):
        if i != 6:
            u = ulist1[i]
            url = url_1 + tplt.format(u[0].encode("utf-8"), chr(255))
            uinfo = []
            html = getHTMLText(url)
            fillUnivList(uinfo, html)
            printUnivList(uinfo, 10)

main

def main():
    uinfo1 = []
     url =http://cheman.chemnet.com/pesticides/search.cgip=3&f=search&u=&c=cate2&s=&t=&terms=%C9%B1%BE%FA%BC%C1&r=
 html = getHTMLText(url)
    fillnameList(uinfo1, html)

    url_1 = http://cheman.chemnet.com/pesticides/supplier2.cgi?f=plist&exact=dict&mark=&terms2=
    work(uinfo1, url_1, 10)