python四种方式解析网页获取页面中的链接

# 链接解释器
# 使用三种不同解释器 环境python3.8
#
from html.parser import HTMLParser
from urllib.request import urlopen
from urllib.parse import urljoin
import time
# 非标准库
from bs4 import BeautifulSoup, SoupStrainer

# 非标准库
# from html5lib import html5parser, treebuilders,treewalkers

# 要解析的网站
URLS = (http://python.org,)


# 定义输出函数
def output(x):
    print(
.join(sorted(list(set(x)))))


def simpleBS(url, f):
    """
    使用BeautifulSoup库来解析所有标签以获得锚点
    :param url: url
    :param f: 文件数据
    :return:
    """
    list_href = [x for x in BeautifulSoup(f).findAll(a)]
    output(urljoin(url, x[href]) for x in list_href)


def fasterBS(url, f):
    """
    使用BeautifulSoup来只解析a标签
    :param url: url
    :param f: 文加数据
    :return:
    """
    list_href = [x for x in BeautifulSoup(f, parse_only=SoupStrainer(a))][1:]
    output(urljoin(url, x[href]) for x in list_href)


def htmlparser(url, f):
    """
    自定义HTMLParser 子类来解析锚点
    :param url: url
    :param f: 文件数据
    :return:
    """

    class AnchorParser(HTMLParser):
        def handle_starttag(self, tag, attrs):
            if tag != a:
                return
            # 若没有data值，增加data属性
            if not hasattr(self, data):
                self.data = []
            for attr in attrs:
                if attr[0] == href:
                    self.data.append(attr[1])

    parser = AnchorParser()
    # 只能feed字符串，故将字节串转化为字符串
    parser.feed(data=f.decode())
    output(urljoin(url, x) for x in parser.data)


def html5libparse(url, f):
    """
    使用html5lib库作为bs的解释器来解析锚点
    :param url: url
    :param f: 文件数据
    :return:
    """
    list_href = [x for x in BeautifulSoup(f.decode(), html5lib).findAll(a)]
    output(urljoin(url, x[href]) for x in list_href)


def process(url, data):
    print(
*** simple BS)
    start = time.time()
    simpleBS(url, data)
    print(simple BS:%.3fs used% (time.time() - start))
    print(
*** faster BS)
    start = time.time()
    fasterBS(url, data)
    print(faster BS %.3fs used% (time.time() - start))
    print(
*** HTMLParse)
    start = time.time()
    htmlparser(url, data)
    print(HTMLParse %.3fs used % (time.time() - start))
    print(
*** HTML5lib)
    start = time.time()
    html5libparse(url, data)
    print(HTML5lib %.3fs used% (time.time() - start))

    # 主函数


def main():
    for url in URLS:
        f = urlopen(url)
        data = f.read()
        f.close()
        # 处理数据
        process(url, data)


main()