安居客 楼盘信息 项目代码-

安居客–抓取楼盘信息(分析加代码) 使用scrapy框架

字段描述 430820 每个楼盘id

爬虫项目结构

spider.py代码如下

# -*- coding: utf-8 -*-
import scrapy
import re


class AjkSpider(scrapy.Spider):
    name = Ajk
    allowed_domains = [zz.fang.anjuke.com]
    start_urls = [http://zz.fang.anjuke.com/]
    info_url = https://zz.fang.anjuke.com/loupan/canshu-{}.html?from=loupan_index_more


    def start_requests(self):
        url = https://zz.fang.anjuke.com/loupan/all/p1_s6/
        yield scrapy.Request(url=url)


    def parse(self, response):
        house_list = response.xpath("//div[@class=key-list]/div")
        for house in house_list:
            house_info_link = house.xpath(./div[@class="infos"]/a[1]/@href).extract_first()
            id = re.findall(rloupan/(.*?).html,house_info_link)[0]
            yield scrapy.Request(url=self.info_url.format(id),callback=self.parse_info)

        try:
            next_link = response.xpath(//div[@class="list-page"]/div[@class="pagination"]/a[text()="下一页"]/@href).extract_first()
            print(下一页: ,next_link)
            yield scrapy.Request(url=next_link,callback=self.parse)
        except Exception as e:
            print(e,没有下一页连接)

    def parse_info(self,response):
        item = {}
        # 基本信息
        item[楼盘名称] = response.xpath(//div[@class="can-border"]/ul/li[1]/div/a/text()).extract_first()
        item[楼盘在售状态] = response.xpath(//div[@class="can-border"]/ul/li[1]/div/i/text()).extract_first()

        li_list1 = response.xpath(//div[@class="can-left"]/div[1]//ul/li)
        for li in li_list1[1:-1]:
            key = li.xpath(./div[1]/text()).extract_first()
            value = .join([i.replace( , ) for i in li.xpath(./div[2]//text()).extract()])
            item[key] = value

        li_list2 = response.xpath(//div[@class="can-left"]/div[2]//ul/li)
        for li in li_list2:
            key = li.xpath(./div[1]/text()).extract_first()
            value = .join([i.replace( , ) for i in li.xpath(./div[2]//text()).extract()])
            item[key] = value

        li_list3 = response.xpath(//div[@class="can-left"]/div[3]//ul/li)
        for li in li_list3:
            key = li.xpath(./div[1]/text()).extract_first()
            value = .join([i.replace( , ) for i in li.xpath(./div[2]//text()).extract()])
            item[key] = value

        yield item


middlewares.py

pipelines.py

import os import csv import pandas as pd

class AjkPipeline(object): def init(self): self.df = pd.DataFrame()

def process_item(self, item, spider):
    print(item)
    self.df = self.df.append(item,ignore_index=True)

def close_spider(self,spider):
    self.df.to_csv(郑州楼盘信息.csv)

class Pipeline_ToCSV(object):

def __init__(self):
    # csv文件的位置,无需事先创建
    store_file = os.path.dirname(__file__) + /spiders/qtw.csv
    # 打开(创建)文件
    self.file = open(store_file, w)
    # csv写法
    self.writer = csv.writer(self.file)

# 将字典的values写入
def process_item(self, item, spider):
    # 判断字段值不为空再写入文件
    print(item)
    self.writer.writerow(item)
    return item

def close_spider(self, spider):
    # 关闭爬虫时顺便将文件保存退出
    self.file.close()