洛阳建设银行官方网站_深圳数字展厅设计公司_建造师培训_重庆交易中心
import requestsfrom bs4 import BeautifulSoupimport timefrom pymongo import MongoClientclass HouseSpider:def __init__(self):self.client = MongoClient('mongodb://localhost:27017/')self.zfdb = self.client.zfdbsession = requests.Session()baseUrl = "http://sz.zu.fang.com"# 每个区域的urlurlDir = {"不限": "/house/","宝安": "/house-a089/","龙岗": "/house-a090/","南山": "/house-a087/","福田": "/house-a085/","罗湖": "/house-a086/","盐田": "/house-a088/","龙华区": "/house-a013080/","坪山区": "/house-a013081/","光明新区": "/house-a013079/","大鹏新区": "/house-a013082/","惠州": "/house-a013058/","东莞": "/house-a013057/","深圳周边": "/house-a016375/",}region = "不限"page = 100# 通过名字获取 url 地址def getRegionUrl(self, name="宝安", page=10):urlList = []for index in range(page):if index == 0:urlList.append(self.baseUrl + self.urlDir[name])else:urlList.append(self.baseUrl + self.urlDir[name] + "i3" + str(index + 1) + "/")return urlList# MongoDB 存储数据结构def getRentMsg(self, title, rooms, area, price, address, traffic, region, direction):return {"title": title, # 标题"rooms": rooms, # 房间数"area": area, # 平方数"price": price, # 价格"address": address, # 地址"traffic": traffic, # 交通描述"region": region, # 区、（福田区、南山区）"direction": direction, # 房子朝向（朝南、朝南北）}# 获取数据库 collectiondef getCollection(self, name):zfdb = self.zfdbif name == "不限":return zfdb.rentif name == "宝安":return zfdb.baoanif name == "龙岗":return zfdb.longgangif name == "南山":return zfdb.nanshanif name == "福田":return zfdb.futianif name == "罗湖":return zfdb.luohuif name == "盐田":return zfdb.yantianif name == "龙华区":return zfdb.longhuaquif name == "坪山区":return zfdb.pingshanquif name == "光明新区":return zfdb.guangmingxinquif name == "大鹏新区":return zfdb.dapengxinqu#def getAreaList(self):return ["不限","宝安","龙岗","南山","福田","罗湖","盐田","龙华区","坪山区","光明新区","大鹏新区",]def getOnePageData(self, pageUrl, reginon="不限"):rent = self.getCollection(self.region)self.session.headers.update({">