From f8848c75acc8df2e8b8eb7d314b110bdbf9839e3 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Wed, 5 Dec 2018 18:11:18 +0800 Subject: [PATCH] fix bug --- PriceSystem/fangjia/ElasticsearchClient.py | 23 --------- PriceSystem/fangjia/__init__.py | 54 ++-------------------- PriceSystem/rent_price/district_map0413.py | 8 ++-- PriceSystem/rent_test.py | 4 +- 4 files changed, 11 insertions(+), 78 deletions(-) delete mode 100644 PriceSystem/fangjia/ElasticsearchClient.py diff --git a/PriceSystem/fangjia/ElasticsearchClient.py b/PriceSystem/fangjia/ElasticsearchClient.py deleted file mode 100644 index 3cbe793..0000000 --- a/PriceSystem/fangjia/ElasticsearchClient.py +++ /dev/null @@ -1,23 +0,0 @@ -from elasticsearch import Elasticsearch -from pypinyin import lazy_pinyin -from fangjia import mongoClients as mongoClients - -city_province_map = {} -es = Elasticsearch(['192.168.0.18']) - -server_area = mongoClients.MongoClients("fangjia", "server_area", "mongo.db.fangjia.com").collection -for area in server_area.find({"display": {'$ne': "全国"}}, {"display": 1, "province": 1}): - city_province_map[area["display"]] = area["province"] - - -def to_pinyin(hans): - return "".join(lazy_pinyin(hans)) - - -def get_index_name(city, region): - big_city = ["北京", "上海", "广州", "深圳", "天津", "杭州", "成都", "苏州", "重庆", "苏州", "南京", "武汉", "厦门"] - if big_city.count(city) > 0: - return "seawater-" + to_pinyin(city) + "-" + to_pinyin(region) - if list(city_province_map.keys()).count(city) == 0: - return "seawater-unknown" - return "seawater-" + to_pinyin(city) diff --git a/PriceSystem/fangjia/__init__.py b/PriceSystem/fangjia/__init__.py index 46a0e43..e1f51e7 100644 --- a/PriceSystem/fangjia/__init__.py +++ b/PriceSystem/fangjia/__init__.py @@ -11,10 +11,11 @@ import pickle import redis from rediscluster import StrictRedisCluster from fangjia import mongoClients as mongoClients -from fangjia import ElasticsearchClient as es from fangjia.cachedObjectWrapper import CachedObjectWrapper from elasticsearch import Elasticsearch, helpers + +import pandas as pd formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s') client_DB_es = Elasticsearch('192.168.0.206', port=9200, timeout=1000, http_auth=('developer', '5e8edd851d2fdfbd7415232c67367cc3')) ES_db_index = "seawater-2018,seawater-2017,seawater-2016" @@ -105,10 +106,6 @@ def get_collection(db, name, host="192.168.0.136", port=27017): return mongoClients.MongoClients(db, name, host, port).collection -def get_seawater_index_name(city, region): - return es.get_index_name(city, region) - - def get_all_city(): server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port()) city_list = [] @@ -178,46 +175,6 @@ def findHouse(query, fields=None): raise Exception("样本查询失败!") -def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000): - index_name = get_seawater_index_name(city, region) - - routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias] - must = [ - { - "term": {"city": city} - }, { - "term": {"cat": cat} - }, { - "term": {"format_region": region} - }, { - "terms": {"district_name.raw": alias} - } - - ] - if roots is not None: - must.append({ - "terms": {"root": roots} - }) - - if pdate is not None: - must.append({"range": {"p_date": {"gt": pdate}}}) - - body = { - - "query": { - "bool": { - "must": must - } - }, - "from": 0, - "size": limit, - "sort": [{"p_date": "desc"}], - "aggs": {} - } - response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body) - return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]]) - - def find_sample_pool(query, fields=None): # s = time.time() seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) @@ -541,16 +498,15 @@ def get_default_redis_connection(): return redis_connection -def findHouse2(query): +def findHouse2(ct, rl, lk, timd, timm): # local_switch and select_content are not applied in this function # query = {"city": ct, "cat": "rent", "region": rl, # 'rent_type':'整租', # "district_name": {"$in": lk}, # "p_date": {"$gt": timd, "$lt": timm}} - city = query['city'] - region = query['region'] + city = ct + region = rl # district_name = lk[0] - name_pack = findquery_statement['name_pack'] query_list = list(helpers.scan( index=ES_db_index, client=client_DB_es, diff --git a/PriceSystem/rent_price/district_map0413.py b/PriceSystem/rent_price/district_map0413.py index 9851ed6..eff082c 100644 --- a/PriceSystem/rent_price/district_map0413.py +++ b/PriceSystem/rent_price/district_map0413.py @@ -175,11 +175,11 @@ def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb): #提取特定时间范围内的小区租金房源信息 def some_district_information(ct,rl,lk,timd,timm): - query = {"city":ct,"cat":"rent","region":rl,#'rent_type':'整租', - "district_name":{"$in":lk}, - "p_date":{"$gt":timd,"$lt":timm}} + # query = {"city":ct,"cat":"rent","region":rl,#'rent_type':'整租', + # "district_name":{"$in":lk}, + # "p_date":{"$gt":timd,"$lt":timm}} - return fangjia.findHouse2(query) + return fangjia.findHouse2(ct, rl, lk, timd, timm) #提取周边小区数据 diff --git a/PriceSystem/rent_test.py b/PriceSystem/rent_test.py index 3eb3733..a61e235 100644 --- a/PriceSystem/rent_test.py +++ b/PriceSystem/rent_test.py @@ -22,8 +22,8 @@ rl='闵行' lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园 #黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期 -timm=20180430 -timd=20170531 +timm=20181130 +timd=20171130 a=time.time() rentPrice.save_to_mongo(ct,rl,lk,timd,timm)