diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 736ff631d..76eed7432 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -832,6 +832,33 @@ class CommonDatautils { } + public static def findNewUpdateDistricts(def city, def cat, def fields){ + + def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + DBObject q = new BasicDBObject(); + q.put("status",0) +// q.put("status",new BasicDBObject('$ne', 0)) + q.put("cat", cat); + q.put("city", city) + q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) +// q.put("loop_line_info",new BasicDBObject('$exists',false)) +// q.put("name","扬州北街8号") + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000) + def rets = [] + for(def d : dbsour){ + rets << d + } + return rets; + } + + public static def findAllRegionDistricts(def city, def region, def cat, def fields){ DBObject q = new BasicDBObject(); @@ -5818,4 +5845,68 @@ class CommonDatautils { } return address; } + + + static def findSeawater(city, lng, lat, distance, p_date, cat){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search" + def body = """ + { + "query": { + "bool": { + "must": [ + { + "geo_distance": { + "distance": "${distance}", + "location": { + "lat": ${lat}, + "lon": ${lng} + } + } + }, + { + "term": { + "city": "${city}" + } + }, + { + "term": { + "cat": "${cat}" + } + }, + { + "range": { + "publish_date": { + "gte": "${p_date}", + "format": "yyyy-MM-dd" + } + } + } + ] + } + }, + "sort": [ + { + "publish_date": { + "order": "desc" + } + } + ], + "size":30 + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = new JsonSlurper().parseText(res) + + def esObj = null + if(response && response.hits){ + esObj = response.hits.hits + } + return esObj + } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy index 52203c79c..7f3f4027b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy @@ -1,21 +1,16 @@ package com.fangjia.script.house -import com.fangjia.core.util.DateUtil + import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DataStore import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject -import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject -import com.mongodb.QueryOperators - - import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -48,14 +43,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{ for(def city : citys){ index ++ ; def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district") - def sellRet = houseStatCount(districts, startDate, endDate, sDate); + + String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd") + + def sellRet = houseStatCount(districts, start.toInteger(), sDate); saveResult(city, sDate, sellRet); println "[${city} ${index}/${citys.size()}]" +new Date().toString(); } } - def houseStatCount(def districts, def startDate, def endDate, def sDate){ + def houseStatCount(def districts, def startDate, def sDate){ def index = 0; @@ -75,8 +73,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{ void run() { index++; def map = [:]; - map.sell_count = findHouseCount(dist, "sell", startDate, endDate); - map.rent_count = findHouseCount(dist, "rent", startDate, endDate); +// map.sell_count = findHouseCount(dist, "sell", startDate, endDate); +// map.rent_count = findHouseCount(dist, "rent", startDate, endDate); + + map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate) + map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate) + map.city = dist.city; map.region = dist.region; map.name = dist.name; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 5a037dba9..5ca528cfb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.jiangnanblank +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -7,24 +8,17 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import com.rabbitmq.client.AMQP -import net.sf.json.JSONObject import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.bson.types.ObjectId -import java.text.DecimalFormat import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong -import java.util.regex.Matcher -import java.util.regex.Pattern - /** * Created by gaojuan on 2017/12/11. * 批量全国小区及商铺字段转换 @@ -33,7 +27,7 @@ import java.util.regex.Pattern class CZDataBatch extends BaseBatch{ public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; - def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"] + def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"] public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl) //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); @@ -47,6 +41,8 @@ class CZDataBatch extends BaseBatch{ @Override void process(Object args) { + println "args 参数:"+args.toString() + def citys = CommonDatautils.findAllCity(); citys.remove("常州"); @@ -57,12 +53,21 @@ class CZDataBatch extends BaseBatch{ def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) def commids = findAllJnBankDatas(city) // - def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); - FzDataUtils.removeNoUpdate(city, districts) + def districts = null; + def shops = null; + if(args && args[0]== "all"){ + districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); + FzDataUtils.removeNoUpdate(city, districts) + }else{ + districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields)); + } districtRun(districts, statMap, commids); - - def shops = CommonDatautils.findAllDistricts(city, "shop", fields); + if(args && args[0]== "all"){ + shops = CommonDatautils.findAllDistricts(city, "shop", fields); + }else{ + shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields); + } shops = mergeShop(shops); shopRun(shops); } @@ -227,11 +232,14 @@ class CZDataBatch extends BaseBatch{ map.build_structure = shop.build_structure // 楼盘均价 map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name); + if(!map.avg_price){ + map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0) + } // 省 市 区 if(shop.city == "常州"){ - map.province_code = regionMap.get("江苏"); - map.city_code = regionMap.get(shop.city); - map.region_code = regionMap.get(shop.region); + map.province_code = FzDataUtils.regionMap.get("江苏"); + map.city_code = FzDataUtils.regionMap.get(shop.city); + map.region_code = FzDataUtils.regionMap.get(shop.region); map.province = "江苏"; map.price_old_string = priceString; map.heights = heights; @@ -259,6 +267,9 @@ class CZDataBatch extends BaseBatch{ map.street_code = FzDataUtils.getStreetCode(street); // 租金 map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name); + if(!map.shop_rent){ + map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0) + } // 物业公司 map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company); // 物业费(每月每平米) @@ -267,6 +278,44 @@ class CZDataBatch extends BaseBatch{ saveShop(map); } + def getSellPrice(def city, def lng, def lat){ + def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd') + def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell") + if(rets){ + def prices = [] + for(def r : rets){ + prices << r._source.avg_price + } + if(prices){ + def price = (int)(prices.sum()/prices.size()) + return price + } + } + return null; + } + + def getRentPrice(def city, def lng, def lat){ + def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd') + def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent") + if(!rets){ + rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent") + if(!rets){ + rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent") + } + } + if(rets){ + def prices = [] + for(def r : rets){ + prices << r._source.avg_price + } + if(prices){ + def day_price = (prices.sum()/prices.size())/30 + return DataHandleUtils.getDoubleRound(2, day_price) + } + } + return null; + } + def saveShop(def m){ DBObject q = new BasicDBObject(); q.put("city",m.city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index 51e58e28e..0c5917dfd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -809,4 +809,5 @@ class FzDataUtils { } + }