diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index d8e1a8ece..950a6a697 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -34,7 +34,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"]) - def rows = createDatas(cityCodeDatas, dists) + def rows = createDatas(cityCodeDatas.get(city), dists) rets.addAll(rows) println city +" "+rows.size() } @@ -44,7 +44,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } - def createDatas(cityCodeDatas, dists){ + def createDatas(cityCode, dists){ def rets = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -54,7 +54,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - def ret = createData(cityCodeDatas, dist) + def ret = createData(cityCode, dist) if(ret){ rets << ret } @@ -72,9 +72,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ return rets; } - def createData(cityCodeDatas, dist){ + def createData(cityCode, dist){ def ret = [] - ret << cityCodeDatas.city_code + ret << cityCode ret << dist._id.toString() ret << CcbCodeUtils.parseBuildType(dist.build_type) ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 84ac44d09..37b58553a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -124,8 +124,6 @@ class BigdataDealBatch extends BaseBatch{ def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; - }else{ - println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() @@ -262,11 +260,11 @@ class BigdataDealBatch extends BaseBatch{ def converData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] - def data = distCodeMap.get(trade.fjRegion) + def data = distCodeMap.get(trade.region) if(!data){ return ret }else{ - println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" + println trade.city +" "+trade.region+" 没有找到对应的区域" } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() @@ -399,7 +397,7 @@ class BigdataDealBatch extends BaseBatch{ def startDate = null; if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) }else{ startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) } @@ -415,7 +413,7 @@ class BigdataDealBatch extends BaseBatch{ if(trades){ tradesList.addAll(trades) } - println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString() + println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString() } catch (Exception e) { e.printStackTrace() } finally { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 52e7de415..03c419ca5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -64,7 +64,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["陵水"]//CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ @@ -157,10 +157,10 @@ class BigdataForsaleBatch extends BaseBatch{ FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); println "txt文件生成 " -// def rets = [] -// rets << headers -// rets.addAll(allDatas) -// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据") + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 1a5bc0a41..cff67d153 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -12,7 +12,7 @@ import com.mongodb.DBObject class CcbCommonUtils { - public static def citys = ["北京", "上海","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + public static def citys = ["上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -185,18 +185,18 @@ class CcbCommonUtils { if(region){ q.put("region", region); } - q.put("name", name); q.put("cat", cat) q.put("status", 0) - q.put("name_cat", 0); - q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) - - DSCursor cursor = getDataStoreConnection().find(q); if(cursor){ - for(def obj : cursor){ - return obj; + for(def d : cursor){ + if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){ + if(d.address){ + d.address = d.address.replaceAll("\r|\n|\\s", ""); + } + return d; + } } } return null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index be208aebb..5a0c7a328 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -52,7 +52,7 @@ class EvaluateRateBatch extends BaseBatch{ } private def findEvaluationRate(){ - def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'"; + def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'"; def rets = [] sqlInstance.eachRow(sql,{ rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index f4b248b9c..a81e03ba1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -66,11 +66,11 @@ class ThirdPartyPriceBatch extends BaseBatch { def data = distCodeMap.get(d.region) if(data){ def ret = [] - ret << third._id.toString() + ret << third.id.toString() ret << d._id.toString() ret << d.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city + ret << cityCodeDatas.get(d.city) + ret << d.city ret << data.district_code ret << data.district ret << third.s_date @@ -177,6 +177,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def dataMap = map.get(key) if(!dataMap){ dataMap = [:] + dataMap.put("id", db._id.toString()) map.put(key, dataMap) } if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){ @@ -240,7 +241,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def resultMap = [:] map.each {key, value-> - if(value){ + if(value && value.size()>1){ resultMap.put(key, value) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 5b0910fc8..96b55d4c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -15,7 +15,10 @@ class SeaweedDataMatchProcess { if(!rMap.city){ def obj = CommonDatautils.findDistrict(city, region, name, "district") if(!obj){ - obj = CommonDatautils.findDistrictByAlias(city, region, name, "district") + def datas = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(datas){ + obj = datas[0] + } } if(obj){ rMap.estate_type2 = obj.estate_type2 @@ -40,14 +43,14 @@ class SeaweedDataMatchProcess { - if(rMap && rMap.id && region.contains(rMap.region)) { - def obj = CommonDatautils.findDistrictById(rMap.id.toString()); - if (obj) { - rMap.estate_type2 = obj.estate_type2 - } - }else{ - rMap = null - } +// if(rMap && rMap.id && region.contains(rMap.region)) { +// def obj = CommonDatautils.findDistrictById(rMap.id.toString()); +// if (obj) { +// rMap.estate_type2 = obj.estate_type2 +// } +// }else{ +// rMap = null +// } if(!rMap && name && name.contains("·")){ def newName = trimChar(name).replace("·","") rMap = matchFromDB2(city,newName) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index 42d0b1ecc..3653fd95e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -30,14 +30,14 @@ class LocationTest { // System.out.println(str); def task = new LocationTest() -// task.run() + task.run() // task.run2() - task.run3() +// task.run3() // task.run4() } def run(){ - def city = "南昌" + def city = "杭州" def regionPointsMap = findRegionRange(city) def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy index dca12c36a..40b17049e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.price +import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.ExcelUtils @@ -11,32 +12,35 @@ class EvaluationBatch { task.run() } def run(){ - def filePath = "D:\\数据\\农行\\东莞农行测试数据(御景花园18条估值)(人工询价).xlsx" + def filePath = "D:\\数据\\银联\\房价网测试数据.xlsx" def city = "东莞" ExcelUtils.modify(filePath, { data -> if(!data['M']){ // def address = data['C'].replace("\"","").trim() // def rMap = SeaweedDataMatchProcess.doMatch(city, data['K'],data['L'], address ); - def rMap = SeaweedDataMatchProcess.doMatch(city, data['D'],data['E'], null); - if(rMap){ - def json = evaluation(rMap, data['H'], 7, data['I'],null) + def region = RegionData.formatRegion(data['A'], data['B']) + if(region){ + def rMap = SeaweedDataMatchProcess.doMatch(data['A'], region, data['C'], null); + if(rMap){ + def json = evaluation(rMap, data['D'], null, null,null) -// data['J'] = rMap.city -// data['K'] = rMap.region -// data['L'] = rMap.name - if(json){ - def avgPrice = json.avg_price - def totalPrice = json.total_price - - data['N'] = totalPrice - data['M'] = avgPrice - data['O'] = json.credit - data['P'] = rMap.region - data['Q'] = rMap.name + data['E'] = rMap.city + data['F'] = rMap.region + data['G'] = rMap.name + if(json){ + def avgPrice = json.avg_price + def totalPrice = json.total_price + data['H'] = totalPrice + data['I'] = avgPrice + data['J'] = json.credit +// data['P'] = rMap.region +// data['Q'] = rMap.name + } } } + } }, 0); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 841fb2f08..6296d9ba8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -625,7 +625,7 @@ public class OpenApiClient { params.put("token", TOKEN) try { JSONObject result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/evaluate", params) - if (result.getInt("code") == 200) { + if (result && result.getInt("code") == 200) { if (result.containsKey("result")) { // if (result.get("result").get("credit") == 0) { // return null