diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 7d934e984..4da8b5658 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -37,14 +37,14 @@ class DistrictSellStatBatch extends BaseBatch { @Override void process(Object args) { -// def months = DateUtils.getMonthList(202206) -// months = months[0..(months.size()-2)] -// println months -// for(def month : months){ -// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd") -// calcMonthlyPrice(curDate) -// } - calcMonthlyPrice() + def months = DateUtils.getMonthList(202209) + months = months[0..(months.size()-1)] + println months + for(def month : months){ + def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd") + calcMonthlyPrice(curDate) + } +// calcMonthlyPrice() } @@ -70,6 +70,8 @@ class DistrictSellStatBatch extends BaseBatch { curDate = new Date() } def citys = CommonUtils.findAllCity() + citys.remove("上海") + citys.remove("福州") def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate)) @@ -87,22 +89,25 @@ class DistrictSellStatBatch extends BaseBatch { for(def city : citys){ index++ + if(index < 242){ + continue + } def districts = [] if(city == "福州"){ - if(day == 2){ + if(day == 29){ districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() }else{ districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList() } }else{ def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0) -// q.put("name", "翠园楼") - if(day == 2){ +// q.put("name", "越华路小区") +// if(day == 30){ districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000).toList() - }else{ - q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)) - districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000) - } +// }else{ +// q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)) +// districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000) +// } } println index+" "+city +" "+districts.size()+" "+ curDate.toString() @@ -155,7 +160,7 @@ class DistrictSellStatBatch extends BaseBatch { def key = dist.region +"##" + dist.name def weekendPriceObj = weekendPriceMap.get(key) - if(weekendPriceObj){ + if(weekendPriceObj && weekendPriceObj.avg_price){ def curPriceObj = curMonthlyPriceMap.get(key) def lastMonthPriceObj = lastMonthlyPriceMap.get(key) def lastYearPriceObj = lastYearPriceMap.get(key) @@ -216,8 +221,19 @@ class DistrictSellStatBatch extends BaseBatch { data.put("credit", weekendPriceObj.credit) def q = new BasicDBObject() - q.put("date", curMonth) - q.put("commid", dist._id.toString()) + if(curPriceObj.commid){ + q.put("date", curMonth) + q.put("commid", dist._id.toString()) + }else{ + q.put("date", curMonth) + q.put("city", dist.city) + q.put("region", dist.region) + q.put("district_name", dist.name) + + data.put("commid", dist._id.toString()) + } + + DISTRICT_SELL_STAT.update(q , new BasicDBObject('$set', data),true,false) // DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false) println curMonth+" [update]"+totalCount.get()+" "+ dist.city+" "+dist.name @@ -239,6 +255,8 @@ class DistrictSellStatBatch extends BaseBatch { DISTRICT_SELL_STAT.save(data) println curMonth+" [save]"+totalCount.get()+" "+ dist.city+" "+dist.name } + + } } catch (Exception e) { @@ -281,6 +299,10 @@ class DistrictSellStatBatch extends BaseBatch { if(d.avg_price_2b && d.value_type!=1){ d.avg_price = d.avg_price_2b } + }else{ + if(d.avg_price_2b){ + d.avg_price = d.avg_price_2b + } } priceMap.put(d.region +"##"+d.district_name, d) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 6ded33f82..8026c0447 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -945,4 +945,17 @@ class OpenApiUtils { return null; } + + static def lactionEvaluation(area, lng, lat){ + def url = "http://192.168.0.100:30020/avmWeb/avmAnn?area=${area}&location=lng"+","+lat + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.msgCode ==200 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy index 4693542de..4b5e0f2e3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy @@ -36,7 +36,8 @@ class SuzouPocTest { // task.newRun() // task.test()//有楼层 // task.testEval()//无楼层 - task.level()//楼盘评级 +// task.level()//楼盘评级 + task.house() } def test(){ @@ -97,36 +98,73 @@ class SuzouPocTest { } def testEval(){ - ExcelUtils.modify("D:\\数据\\其它\\附件1:POC测试样本(匹配).xlsx", { d -> - if(d['O'] && !d['T']){ + ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d -> + if(!d['O'] && d['E']){ // def city = d['C'] // if(city == "市辖区"){ // city = d['B'] // } - def obj = CommonDatautils.findDistrictById(d['O']) - if(obj){ - d['R'] = obj.city - d['S'] = obj.region - d['T'] = obj.name - def ret1 = OpenApiClient.addressEvaluation(obj.city, obj.region, obj.city+obj.region+obj.name, obj.name, d['F'], null, null) +// def obj = CommonDatautils.findDistrictById(d['O']) +// if(obj){ +// d['R'] = obj.city +// d['S'] = obj.region +// d['T'] = obj.name + def ret1 = OpenApiClient.addressEvaluation(d['C'], d['D'], d['C']+d['D']+d['E'], d['E'], d['I'], null, null) if (ret1) { - d['P'] = ret1.totalPrice - d['Q'] = ret1.avgPrice - }else{ - def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name) - if(dbo) { - d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble()) - d['Q'] = dbo.avg_price_2b - - } - + d['J'] = ret1.totalPrice + d['K'] = ret1.avgPrice + d['L'] = ret1.city + d['M'] = ret1.district + d['N'] = ret1.name } - } +// else{ +// def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name) +// if(dbo) { +// d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble()) +// d['Q'] = dbo.avg_price_2b +// +// } +// +// } +// } } }); } + def house(){ + def cases = [] + cases << ["城市", "区域","板块", "名称", "面积", "单价","总价","装修", "朝向", "距离","所在楼层","总楼层","室","厅","卫","发布时间","标题", "序号"] + ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d -> + if(d['L']){ + def obj = CommonDatautils.findDistrict(d['L'], d['M'], d['N'], "district") + if(obj){ + def lng = obj.lng/10000000000.0 + def lat = obj.lat/10000000000.0 + def location = String.valueOf(lng)+","+ String.valueOf(lat) + def result = OpenApiClient.houseAroundBylocationAction(location, d['I'], 0, 0, 50) + if(result){ + for(def h : result){ + cases << [h.city, h.district, h.block, h.name, h.size, h.unitPrice, h.totalPrice, h.decoration, h.direction, h.distance, h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.publishDate, h.title, d["A"],"挂牌"] + } + } + def address = d['L']+d['M']+d['N'] + def dealRets = OpenApiClient.dealHouse(d['L'], d['M'], d['N'], address, d['I']) + if(dealRets){ + for(def h : dealRets){ + cases << [h.city, h.district, "", h.name, h.size, h.avgPrice, h.totalPrice, h.decoration, h.direction, "", h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.tradeDate, "", d["A"], "成交"] + } + } + + } + + } + + }) + println cases.size() + ExcelUtils.write("D:\\数据\\宏信息\\样本房源-估值.xlsx", cases, "样本") + } + def newRun(){ println("start:"+new Date().toString()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 50ec412b8..45332f6f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -33,13 +33,11 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx -// task.addDev(path) - task.addDevAddress(path) + task.addDev(path) +// task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") - - } def addDev(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index ea28d2c0b..70b407ceb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -191,7 +191,7 @@ class NewGaoDePoiBatch extends BaseBatch{ if(smallArr && smallArr.size()>=1){ saveObj.small_type = smallArr[0] }else{ - saveObj.small_type = smallstr + saveObj.8 = smallstr } }else{ saveObj.small_type = saveObj.secondary_type diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 3797a3924..8edaa8a99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -57,9 +57,9 @@ class TownStandardBatch extends BaseBatch{ } def TownStandard(){ - townStandardByCat("office") +// townStandardByCat("office") // townStandardByPoi() -// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 + addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 // standardTown() // updateTownData() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy new file mode 100644 index 000000000..04ec66d5f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId + +class DistrictLocationGeo { + + final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) + + static main(args) { + def task = new DistrictLocationGeo() + task.run()//调用高德逆解析,并保存数据 + } + def run(){ + ExcelUtils.modify("D:\\数据\\京东\\需逆解析数据-test.xlsx",{ d-> + def jsonObject = GaoDeApiUtils.geocode(d['C']) + if(jsonObject){ + def dbo = new BasicDBObject() + dbo.put("old_id", d['B']) + dbo.put("old_location", d['C']) + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + dbo.put("addressComponent", jsonObject.regeocode.addressComponent) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + seaweedGaodePoisColl.save(dbo) + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy index 8621a2787..d4bff37bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy @@ -20,10 +20,12 @@ class TestEvaluation { // task.match() // task.houseSimilar() - task.newMatch() +// task.newMatch() // task.houseSimilar() - task.newAndOldMatch() +// task.newAndOldMatch() + + task.addFjwToBprice("D:\\数据\\估值\\地址估价-匹配(加写字楼-商铺) - v2.xlsx") } def houseSimilar(){ @@ -206,4 +208,19 @@ class TestEvaluation { }) } + def addFjwToBprice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ + def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) + if(dbo){ + if(dbo.avg_price_2b){ + d['H'] = dbo.avg_price_2b + } + } + } + }) + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 9d7fad788..c3dbfe651 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -8,6 +8,7 @@ import com.mongodb.DBCollection import com.mongodb.DBCursor import net.sf.json.JSONArray import net.sf.json.JSONObject +import net.sf.json.groovy.JsonSlurper import org.apache.commons.lang.StringUtils import org.apache.commons.lang.SystemUtils @@ -1577,5 +1578,73 @@ public class OpenApiClient { + public static def houseAroundBylocationAction(location, size, propertyType, houseType, pageSize) { + + def url ="https://open.fangjia.com/house/around?location="+location+"&size="+size+"&pageSize="+pageSize+"&propertyType="+propertyType+"&houseType="+houseType + url = url +"&token="+TOKEN + + def content = new HttpRequest().newGet(url, null, "utf-8") + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dtos = doc.result + return dtos + } + } + + return null + } + + + static def dealHouse(city, region, name, area, address){ + + def url ="https://open.fangjia.com/property/deal?city="+city+"&address="+address+"&district="+region+"&name="+name+"&size="+area + url = url +"&token="+TOKEN + + def content = new HttpRequest().newGet(url, null, "utf-8") + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dtos = doc.result + return dtos + } + } + + return null + } + + + public static JSONObject propertyHouseAction(city, district, name, size, propertyType) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("district", district) + params.put("name", name) + params.put("size", size.toString()); + params.put("token", TOKEN); + params.put("propertyType", propertyType) + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/property/house", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/property/house", params, "utf-8") + println("${city} ${name} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 0) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${name} ") + } + } catch (Exception e) { + println("${city} ${name} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 60dc4ddb2..c9dceabe6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -13,7 +13,7 @@ import java.util.zip.GZIPInputStream class GaoDeApiUtils { //94c74c60e102972a8d17b7b75d7403be - def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 + def static GAO_DE_AK = "a34894d151fffff94f9a33bae3d5565c" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 //15021384995:dfc2245bac8a7b2d5b62e060e59d4eca /** @@ -29,6 +29,8 @@ class GaoDeApiUtils { params.put("key", GAO_DE_AK) params.put("location", location) params.put("extensions","all") + params.put("radius", "3000") + params.put("poitype", "120000|120200|120203|120300|120301|120302|120303") String content def stop = true JSONObject jsonObj = null