This commit is contained in:
2018-12-05 18:11:18 +08:00
parent 6c21676583
commit f8848c75ac
4 changed files with 11 additions and 78 deletions
@@ -1,23 +0,0 @@
from elasticsearch import Elasticsearch
from pypinyin import lazy_pinyin
from fangjia import mongoClients as mongoClients
city_province_map = {}
es = Elasticsearch(['192.168.0.18'])
server_area = mongoClients.MongoClients("fangjia", "server_area", "mongo.db.fangjia.com").collection
for area in server_area.find({"display": {'$ne': "全国"}}, {"display": 1, "province": 1}):
city_province_map[area["display"]] = area["province"]
def to_pinyin(hans):
return "".join(lazy_pinyin(hans))
def get_index_name(city, region):
big_city = ["北京", "上海", "广州", "深圳", "天津", "杭州", "成都", "苏州", "重庆", "苏州", "南京", "武汉", "厦门"]
if big_city.count(city) > 0:
return "seawater-" + to_pinyin(city) + "-" + to_pinyin(region)
if list(city_province_map.keys()).count(city) == 0:
return "seawater-unknown"
return "seawater-" + to_pinyin(city)
+5 -49
View File
@@ -11,10 +11,11 @@ import pickle
import redis
from rediscluster import StrictRedisCluster
from fangjia import mongoClients as mongoClients
from fangjia import ElasticsearchClient as es
from fangjia.cachedObjectWrapper import CachedObjectWrapper
from elasticsearch import Elasticsearch, helpers
import pandas as pd
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
client_DB_es = Elasticsearch('192.168.0.206', port=9200, timeout=1000, http_auth=('developer', '5e8edd851d2fdfbd7415232c67367cc3'))
ES_db_index = "seawater-2018,seawater-2017,seawater-2016"
@@ -105,10 +106,6 @@ def get_collection(db, name, host="192.168.0.136", port=27017):
return mongoClients.MongoClients(db, name, host, port).collection
def get_seawater_index_name(city, region):
return es.get_index_name(city, region)
def get_all_city():
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
city_list = []
@@ -178,46 +175,6 @@ def findHouse(query, fields=None):
raise Exception("样本查询失败!")
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
index_name = get_seawater_index_name(city, region)
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
must = [
{
"term": {"city": city}
}, {
"term": {"cat": cat}
}, {
"term": {"format_region": region}
}, {
"terms": {"district_name.raw": alias}
}
]
if roots is not None:
must.append({
"terms": {"root": roots}
})
if pdate is not None:
must.append({"range": {"p_date": {"gt": pdate}}})
body = {
"query": {
"bool": {
"must": must
}
},
"from": 0,
"size": limit,
"sort": [{"p_date": "desc"}],
"aggs": {}
}
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
def find_sample_pool(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
@@ -541,16 +498,15 @@ def get_default_redis_connection():
return redis_connection
def findHouse2(query):
def findHouse2(ct, rl, lk, timd, timm):
# local_switch and select_content are not applied in this function
# query = {"city": ct, "cat": "rent", "region": rl, # 'rent_type':'整租',
# "district_name": {"$in": lk},
# "p_date": {"$gt": timd, "$lt": timm}}
city = query['city']
region = query['region']
city = ct
region = rl
# district_name = lk[0]
name_pack = findquery_statement['name_pack']
query_list = list(helpers.scan(
index=ES_db_index,
client=client_DB_es,
+4 -4
View File
@@ -175,11 +175,11 @@ def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb):
#提取特定时间范围内的小区租金房源信息
def some_district_information(ct,rl,lk,timd,timm):
query = {"city":ct,"cat":"rent","region":rl,#'rent_type':'整租',
"district_name":{"$in":lk},
"p_date":{"$gt":timd,"$lt":timm}}
# query = {"city":ct,"cat":"rent","region":rl,#'rent_type':'整租',
# "district_name":{"$in":lk},
# "p_date":{"$gt":timd,"$lt":timm}}
return fangjia.findHouse2(query)
return fangjia.findHouse2(ct, rl, lk, timd, timm)
#提取周边小区数据
+2 -2
View File
@@ -22,8 +22,8 @@ rl='闵行'
lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园
#黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期
timm=20180430
timd=20170531
timm=20181130
timd=20171130
a=time.time()
rentPrice.save_to_mongo(ct,rl,lk,timd,timm)