Merge branch 'script_gj' into 'master'

无经纬度不同步

See merge request !85
This commit is contained in:
2021-08-19 19:00:26 +08:00
25 changed files with 1158 additions and 173 deletions
@@ -33,7 +33,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def rows = createDatas(cityCodeDatas.get(city), dists)
rets.addAll(rows)
println city +" "+rows.size()
@@ -390,7 +390,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","build_type","build_structure", "house_property","property_life"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -64,14 +64,14 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = ["陵水"]//CcbCommonUtils.citys;
def citys = ["陵水","文昌","保亭"]//CcbCommonUtils.citys;
def index = 0
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
}
}
@@ -0,0 +1,195 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 建行数据poi同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataPoiBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
runBatch(new BigdataPoiBatch(), args);
}
@Override
public void process(Object args) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def rets = []
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
rets.addAll(rows3)
//
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt"
}
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "txt 生成"
// def headers1 = ["城市ID","城市","行政区ID","区域名称","经度","纬度","创建时间","来源公司"]
// def regionCenterDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\行政区中心点经纬度.xlsx", 0,1)
// def regionRets = createRegionCenters(regionCenterDatas)
// FileUtils.batchCreateFile(regionRets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\region_center.txt", headers1, "utf-8", false);
// println "行政区域中心txt 生成"
}
def createRegionCenters(def datas){
def rets = []
for(def d : datas){
def ret = []
ret << d['A'].toInteger().toString()
ret << d['B']
ret << d['C'].toInteger().toString()
ret << d['D']
def loactionStr = d['E']
if(loactionStr){
def arrs = loactionStr.split(",")
def loactionMap = GeoUtils.gcjToGps84(arrs[1].toDouble(), arrs[0].toDouble())
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
}else{
ret << ""
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
return rets;
}
def createDatas(citys, cityCodeDataMap, datas, type){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def cityDistMap = CcbCommonUtils.findAllCityDistrictCodeValue()
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(citys.contains(d['D'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){
rets << ret
}
}
println totalCount.get()+" "+ d['D']+" "+d['F']
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['D'])
ret << citycode
def distCodeMap = cityDistMap.get(d['D'])
if(d['E']){
def distObj = distCodeMap.get(d['E'])
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << d['A']
ret << d['F']
ret << type
if(type=="地铁"){
ret << "cz125"
}else if(type=="公交"){
ret << "cz123"
}else if(type=="医院"){
ret << "hp111"
}else if(type == "银行"){
ret << "se118"
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['B'].contains("141202")){
ret << "ed138"
}else if(d['B'].contains("141203")){
ret << "ed114"
}
}
if(d['I']){
def lat = d['I'].toDouble()
def lng = d['H'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
return ret;
}
}
@@ -12,7 +12,8 @@ import com.mongodb.DBObject
class CcbCommonUtils {
public static def citys = [ "扬州","上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
//
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -47,6 +48,21 @@ class CcbCommonUtils {
return map;
}
public static def findAllCityDistrictCodeValue(city){
def q = new BasicDBObject()
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
def distMap = map.get(db.fangjia_city)
if(!distMap){
distMap = [:]
map.put(db.fangjia_city, distMap)
}
distMap.put(db.fangjia_district, db)
}
return map;
}
public static def findPersonPrice(city, date){
def q = new BasicDBObject()
q.put("city", city)
@@ -170,7 +186,7 @@ class CcbCommonUtils {
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
@@ -27,7 +27,7 @@ class StagingDistrictBatch extends BaseBatch{
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","master_district_name", "c_date", "m_date"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap)
rows.addAll(rets)
@@ -22,10 +22,6 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
//String str="\${ }";
//String regex="\\\$\\{.*\\}";
//Matcher m=Pattern.compile(regex).matcher(str);
//System.out.println(m.matches());
@@ -63,7 +59,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
@@ -79,48 +76,58 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
// dealMap.put(d['A'], map)
//}
//def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//ExcelUtils.multiThreadingModify(fileName,{d->
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//小区量
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def city = d['B']
// if(city){
// if (city) {
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
//// d['I'] = total
//// def a = findEstateType(cityid, sqlInstance)
////
//// d['AA'] = total
//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
////
//// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
////
//// def c = findFiledCount(cityid, 'address', sqlInstance)
//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
////
//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
////
//// def e = findFiledCount(cityid, 'landarea', sqlInstance)
//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
////
//// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
////
//// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
////
//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
////
//// def i = findFiledCount(cityid, 'latitude', sqlInstance)
//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
////
//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
//
// println d
// d['K'] = total
// }
//},0)
//小区字段覆盖率
ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['A']
if(city){
def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance)
d['AA'] = total
def a = findEstateType(cityid, sqlInstance)
d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
def b = findFiledCount(cityid, 'buildyear', sqlInstance)
d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
def c = findFiledCount(cityid, 'address', sqlInstance)
d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
def e = findFiledCount(cityid, 'landarea', sqlInstance)
d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
def f = findFiledCount(cityid, 'buildarea', sqlInstance)
d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
def g = findFiledCount(cityid, 'plotratio', sqlInstance)
d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
def i = findFiledCount(cityid, 'latitude', sqlInstance)
d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
println d
}
},1)
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
//def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d->
@@ -206,64 +213,64 @@ def findFiledCount(cityid, field, sqlInstance){
}
def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
def updateIndex = 0
def delIndex = 0
def cityMap = [:]
ExcelUtils.multiThreadingModify(fileName,{d->
if(!d['R']){
def obj = CommonDatautils.findDistrictById(d['D'])
if(obj){
def ranges = cityMap.get(obj.city)
if(!ranges){
def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
ranges = []
for(def r : dbcursor){
ranges << r
}
cityMap.put(obj.city, ranges)
}
if(ranges){
def lng = obj.lng/10000000000.0
def lat = obj.lat/10000000000.0
def region = null
for(def r : ranges){
try{
def flag = isInArea(lng, lat, r.polyline)
if(flag){
def regionStr = r.region
region = RegionData.formatRegion(obj.city, regionStr)
break
}
}catch(Exception e){
println e.getMessage()
}
}
if(region && region!=obj.region){
updateIndex++
DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
}else if(region && region==obj.region){
println "[same] "+obj.city +" "+obj.name
}else{
DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
delIndex++
println delIndex +" [delete] "+obj.city +" "+obj.name
}
}else{
println obj.city +" 没有此城市边框信息"
}
}else{
println d['D']+" 没找到该小区"
}
}
},0);
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
//def updateIndex = 0
//def delIndex = 0
//def cityMap = [:]
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(!d['R']){
//
// def obj = CommonDatautils.findDistrictById(d['D'])
// if(obj){
// def ranges = cityMap.get(obj.city)
// if(!ranges){
// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
// ranges = []
// for(def r : dbcursor){
// ranges << r
// }
// cityMap.put(obj.city, ranges)
// }
//
// if(ranges){
// def lng = obj.lng/10000000000.0
// def lat = obj.lat/10000000000.0
//
// def region = null
// for(def r : ranges){
// try{
// def flag = isInArea(lng, lat, r.polyline)
// if(flag){
// def regionStr = r.region
// region = RegionData.formatRegion(obj.city, regionStr)
// break
// }
// }catch(Exception e){
// println e.getMessage()
// }
//
// }
// if(region && region!=obj.region){
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
// }else if(region && region==obj.region){
// println "[same] "+obj.city +" "+obj.name
// }else{
// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
// delIndex++
// println delIndex +" [delete] "+obj.city +" "+obj.name
// }
// }else{
// println obj.city +" 没有此城市边框信息"
// }
//
// }else{
// println d['D']+" 没找到该小区"
// }
//
// }
//},0);
@@ -38,7 +38,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
rows.addAll(rets)
@@ -180,58 +180,58 @@ class ThirdPartyPriceBatch extends BaseBatch {
dataMap.put("id", db._id.toString())
map.put(key, dataMap)
}
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price){
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price){
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price){
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price){
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price")){
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price){
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price){
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price){
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
dataMap.put("qfang_esf_price", db.qfang_esf_price)
}
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price){
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
}
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price){
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
}
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price")){
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
@@ -25,6 +25,7 @@ class TradeStatLog {
def task = new TradeStatLog()
//task.run()
task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
}
@@ -88,7 +89,7 @@ class TradeStatLog {
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"]
for(def city : citys){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat","lng"]);
def m = getThirdPartyPricePrecent(city)
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
println rets.get(rets.size()-1)
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202106.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -1,6 +1,7 @@
package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils
@@ -29,7 +30,8 @@ class RegionPointCrawl {
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
def provinces = ["浙江省"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
@@ -55,7 +57,13 @@ class RegionPointCrawl {
}
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
if(ret.size()>0){
saveData(cityMap.name, m.name, m.ad_code , ret);
def standardCity = RegionData.formatCity(cityMap.name)
if(standardCity){
saveData(standardCity, m.name, m.ad_code , ret);
}else{
println cityMap.name +" 没有标准化"
}
}
}
}
@@ -6,6 +6,7 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
@@ -47,14 +48,13 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-06-01"
static def statDate = "2021-07-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
}
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -308,10 +308,7 @@ class YouDaDataParser {
}
);
}
latch.await()
}
def saveEsfHouseToDB(def data){
@@ -367,15 +364,16 @@ class YouDaDataParser {
row.ydId = data[2].toInteger()
row.fjCity = city
def map = matchDistrict(city, row.region, row.districtName)
if(!map){
map = matchDistrict(city, row.region, row.address)
}
// def map = matchDistrict(city, row.region, row.districtName)
// if(!map){
// map = matchDistrict(city, row.region, row.address)
// }
def map = SeaweedDataMatchProcess.doMatch(city, row.region, row.districtName, row.address);
if(map){
row.fjRegion = map.region
row.fjName = map.name
row.lat = map.lat
row.lng = map.lng
row.lat = map.lat.toDouble()
row.lng = map.lng.toDouble()
row.location = map.lat.toString()+","+map.lng.toString()
def obj = CommonDatautils.findDistrictById(map.id.toString());
@@ -7,16 +7,22 @@ import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class AddDistrictFieldValue {
@@ -30,18 +36,26 @@ class AddDistrictFieldValue {
}
def run(){
def citys = CcbCommonUtils.citys;
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){
// def rets = findAllDataByCity(city)
// match(rets)
// }
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
def rMap = getMatchDistrict(city)
createResultValue(dists, rMap)
}
//第三方均价数据填补
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
// def rMap = getMatchDistrict(city)
// createResultValue(dists, rMap)
// }
//
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
// addFinishDate(dists)
addFieldValueByHouse(dists)
}
}
def match(def rets){
@@ -118,6 +132,35 @@ class AddDistrictFieldValue {
return rMap;
}
def addFinishDate(dists){
def updateIndex = 0
def total = dists.size()
for(def d : dists){
if(!d.finish_building_date){
def udata = new BasicDBObject()
def newValue = FiledStrand.processFinishBuildingDate(d.live_date)
if(!newValue && d.open_date){
def date = d.open_date
newValue = date.getYear()+1900
}
if(!newValue && d.build_date){
newValue = FiledStrand.processFinishBuildingDate(d.build_date)
}
if(newValue){
udata.put("finish_building_date", newValue)
}
if(udata.size()>0){
updateIndex++
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
def createResultValue(dists, rMap){
def updateIndex = 0
def total = dists.size()
@@ -260,9 +303,163 @@ class AddDistrictFieldValue {
}
}
}
return lastValue;
}
def addFieldValueByHouse(def dists){
def updateIndex = new AtomicLong()
def totalCount = new AtomicLong()
def total = dists.size()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(d.estate_type2!="别墅" && (!d.finish_building_date || !d.build_type)){
def udata = new BasicDBObject()
def retMap = findHouses(d.city, d.region, "sell", d.name, d.alias)
if(retMap){
println retMap
if(!d.finish_building_date && retMap.containsKey("build_date")){
def dateMap = retMap.get("build_date")
if(dateMap){
def value = dateMap.keySet()[0]
if(value){
udata.put("finish_building_date", value)
}
}
}
if(!d.build_type && retMap.containsKey("height")){
def heightMap = retMap.get("height")
if(heightMap){
def heights = heightMap.keySet()
if(heights){
def max = heights.max()
if(max<=6){
udata.put("build_type", "多层")
}else if(max>6 && max<=11){
udata.put("build_type", "小高层")
}else if(max>11 && max<30){
udata.put("build_type", "高层")
}else if(max>30){
udata.put("build_type", "超高层")
}
}
}
}
}
if(udata.size()>0){
updateIndex.addAndGet(1)
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex.get() +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}else{
println totalCount.get() +" "+d.city +" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
latch.await(5, TimeUnit.MINUTES);
}
def findHouses(def city, def region, def cat, def name, def alias){
// QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
// must(QueryBuilders.termQuery("cat", cat)).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def restMap = [:]
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(seawater.build_date){
def newValue = FiledStrand.processFinishBuildingDate(seawater.build_date)
if(newValue){
def valueMap = restMap.get("build_date")
if(!valueMap){
valueMap = [:]
restMap.put("build_date", valueMap)
}
def count = valueMap.get(newValue)
if(!count){
valueMap.put(newValue, 1)
}else{
valueMap.put(newValue, count + 1)
}
}
}
if(seawater.height){
def hightMap = restMap.get("height")
if(!hightMap){
hightMap = [:]
restMap.put("height", hightMap)
}
def count = hightMap.get(seawater.height)
if(!count){
count = 1
hightMap.put(seawater.height, count)
}else{
hightMap.put(seawater.height, count + 1)
}
}
}
}
if(restMap){
def dateMap = restMap.get("build_date")
if(dateMap){
dateMap = dateMap.sort {a, b -> b.value <=> a.value}
restMap.put("build_date", dateMap)
}
}
return restMap;
}
}
@@ -24,7 +24,7 @@ class DataProcess {
static main(args) {
//建行信用卡城市(临汾--晋城).xlsx
def filePath = "D:\\数据\\小区\\地图\\张家口.xlsx"
def filePath = "D:\\数据\\小区\\地图\\兰州.xlsx"
def task = new DataProcess();
// task.batchSaveDataToDB(filePath)
task.mapBatchSaveDataToDB(filePath)
@@ -121,7 +121,7 @@ class DataProcess {
if(datas){
def keys = []
for(def data : datas){//20,21,25,
if(data['I'] && [20,21,25,26,27].contains(data['I'].toString().toDouble().toInteger())){
if(data['I'] && [20,21,26,27].contains(data['I'].toString().toDouble().toInteger())){
def key = data['A']+"##"+data['B']+"##"+data['H']
if(!keys.contains(key) && data['H'] && data['H'].size()>2){
keys << key
@@ -5,15 +5,22 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import com.mongodb.BasicDBObject
import java.util.regex.Pattern
/***
* 地图小区处理
*/
class MapDistrictProcess {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) {
def task = new MapDistrictProcess();
task.run()
@@ -28,17 +35,18 @@ class MapDistrictProcess {
def run(){
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
def files = ["青岛.xlsx","沈阳.xlsx","哈尔滨.xlsx","张家口.xlsx","大连.xlsx"]
//"乌鲁木齐.xlsx","烟台.xlsx","保定.xlsx","贵阳.xlsx","湖州.xlsx","惠州.xlsx","吉林.xlsx","遵义.xlsx","宜昌.xlsx","芜湖.xlsx","兰州.xlsx"
def files = ["兰州.xlsx"]
for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
typeFilter(filePath)
nameStandard(filePath);
dataFilter(filePath)
houseCalculate(filePath);
////// //统计是否存在多个区域
//统计是否存在多个区域
matchDistDatas(filePath)
moreRegionCalculate(filePath)
//
addFlag(filePath)
mapMatch(filePath);
baiduTypeFilter(filePath);
@@ -50,7 +58,8 @@ class MapDistrictProcess {
addFeature(filePath)
houseMatch(filePath);
// nameScore(filePath);
nameScore(filePath);
thirdPartyData(filePath)
}
}
@@ -212,7 +221,7 @@ class MapDistrictProcess {
def addFlag(def filePath){
def index = 0
ExcelUtils.modify(filePath, { row ->
ExcelUtils.multiThreadingModify(filePath, { row ->
index++
if(!row['I']) {
//根据别名查找
@@ -438,6 +447,34 @@ class MapDistrictProcess {
},0)
}
def thirdPartyData(filePath){
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['I'] && [20,21,25,26,27].contains(row['I'].toString().toDouble().toInteger())){
def rets = findIsExistsOtherWebsite(row['A'], row['B'], row['H'])
if(rets){
row['W'] = rets.join(",")
}
}
},0)
}
def findIsExistsOtherWebsite(def city, def region, def name){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name", name)
def dbcursor = districtDetailColl.find(q)
def rets = []
for(def r : dbcursor){
if(!rets.contains(r.root)){
rets << r.root
}
}
return rets
}
def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
@@ -25,6 +25,9 @@ class SeaweedDataMatchProcess {
rMap.city = obj.city
rMap.region = obj.region
rMap.name = obj.name
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
rMap.id = obj._id.toString()
}
}
}
@@ -0,0 +1,320 @@
package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
class AddDetailDistrict {
def detailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) {
def task = new AddDetailDistrict();
// task.run();
// task.delRepeatDist
task.addEstateType()
}
def addEstateType(){
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","estate_type2"])
for(def d : districts) {
if(!d.estate_type2){
def estateType2 = "普通住宅"
if(d.name.contains("别墅")){
estateType2 = "别墅"
}else if(d.name.contains("商住")){
estateType2 = "商住"
}
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("estate_type2", estateType2).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
println city
}
}
def delRepeatDist(){
def citys = CcbCommonUtils.citys
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
def index = 0
for(def d : districts){
index++
def key = d.city +"##"+d.region+"##"+d.name
if(!keys.contains(key)){
keys << key
}else{
println index +" "+d.region+" "+d.name
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("status", 1).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
}
}
def run() {
def citys = ["沈阳", "哈尔滨", "吉林", "遵义", "淄博", "西宁", "青岛", "大连", "宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
for (def city : citys) {
def rets = findAllDataByCity(city)
def rMap = getMatchDistrict(city)
toDB(rets, rMap)
}
}
def toDB(def rets, def rMap){
def saveDistKeys = []
def total = rets.size()
def index = 0
def addIndex = 0
for(def d : rets){
index++
def newRegion = RegionData.formatRegion(d.city, d.region)
def name = NameProcess.nameStandard(d.city, newRegion, d.name)
if(!name){
continue
}
def flag = NameProcess.filterName(name)
def key = null
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
if(flag && !saveDistKeys.contains(key)){
def gdData = FangjiaClientUtils.getMapData(d.city, newRegion, name, d.address,"", "高德")
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(gdData){
if(!newRegion){
newRegion = RegionData.formatRegion(d.city, gdData.district)
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
}
}
if(key && gdData && gdData.lng){
def matchMap = SeaweedDataMatchProcess.doMatch(d.city, newRegion, name, d.address);
if(!matchMap){
addIndex++
if(gdData.lng){
def gmap = GeocodeUtils.getGaodeToBaidu(gdData.lng.toDouble(), gdData.lat.toDouble())
lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
lng = (gdData.lng.toDouble()* 10000000000).longValue()
lat = (gdData.lat.toDouble()* 10000000000).longValue()
}
def newAddress = DataUtil.formateAddress(d.city, d.address)
def udata = createFieldValue(d, rMap)
udata.put("lng", lng)
udata.put("lat", lat)
println index+"/${addIndex}/${total} "+d.city+" "+newRegion+" "+name+" "+d.root+" "+udata
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", name)
udata.put("address", newAddress)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("status", 0)
udata.put("cat", "district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
DbUtils.seaweed.save(udata)
saveDistKeys << key
}
}
// println d.city+" "+ d.region+" "+d.name
}
}
}
def createFieldValue(d, rMap){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
def udata = new BasicDBObject()
if(sourceMap){
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
udata.put("all_area", lastValue)
}
}
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
udata.put("build_area", lastValue)
}
}
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
udata.put("virescence_percent", lastValue)
}
}
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
udata.put("cubage_percent", lastValue)
}
}
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
udata.put("finish_building_date", lastValue)
}
}
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
udata.put("household_count", lastValue)
}
}
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
udata.put("car_site_count", lastValue)
}
}
}
return udata
}
def getValuesByField(sourceMap, field, estateType){
def map = [:]
sourceMap.each{k,v->
def oldValue = v.get(field);
if(oldValue){
def newValue = null;
if(["address"].contains(field)){
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
}
if(["all_area","build_area"].contains(field)){
newValue = FiledStrand.processArea(oldValue)
}
if(["build_date"].contains(field)){
newValue = FiledStrand.processFinishBuildingDate(oldValue)
}
if(["house_count"].contains(field)){
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
}
if(["greening_rate"].contains(field)){
newValue = FiledStrand.processVirescencePercent(oldValue)
}
if(["volumetric_rate"].contains(field)){
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
}
if(["car_site_desc","car_site"].contains(field)){
newValue = FiledStrand.processCarSite(oldValue)
}
if(newValue){
def count = map.get(newValue)
if(!count){
map.put(newValue, 1)
}else{
map.put(newValue, count+1)
}
v.put(field, newValue)
}
}
}
def lastValue = null
if(map && map.size()>0){
map = map.sort {a, b -> b.value <=> a.value}
def values = map.values()
if(values.size()==1){
lastValue = map.keySet()[0]
}else{
def count1 = map.values()[0]
def count2 = map.values()[1]
if(count1 > count2){
lastValue = map.keySet()[0]
}else{
def rets = [map.keySet()[0],map.keySet()[1]]
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
for(def root : roots){
def data = sourceMap.get(root)
if(data){
def value = data.get(field)
if(rets.contains(value)){
lastValue = value
break
}
}
}
}
}
}
return lastValue;
}
def getMatchDistrict(def city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("match_name", new BasicDBObject('$exists', true))
def dbcursor = detailColl.find(q)
def rMap = [:]
for(def d : dbcursor){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
if(!sourceMap){
sourceMap = [:]
rMap.put(key, sourceMap)
}
def value = sourceMap.get(d.root)
if(!value){
sourceMap.put(d.root, d)
}
}
return rMap;
}
def findAllDataByCity(def city){
def q = new BasicDBObject("city", city);
q.put("match_name", new BasicDBObject('$exists', false))
q.put("avg_price", new BasicDBObject('$gt', 0))
def dbcursor = detailColl.find(q).batchSize(2000)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -1,8 +1,9 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.datastore.FieldValidatorUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.DbUtils
@@ -0,0 +1,27 @@
package com.fangjia.script.gj.location
class Point {
Double lat;
Double lng;
public Point(Double lng, Double lat){
this.lat = lat
this.lng = lng
}
Double getLat() {
return lat
}
void setLat(Double lat) {
this.lat = lat
}
Double getLng() {
return lng
}
void setLng(Double lng) {
this.lng = lng
}
}
@@ -0,0 +1,143 @@
package com.fangjia.script.gj.location
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.awt.geom.Point2D
class PointUtil {
public static final DBCollection BUSINESS_RANGE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border", "mongo.db.fangjia.com")
/**
* 判断当前位置是否在多边形区域内
*
* @param checkPoint 当前经纬度
* @param areaAround 围栏范围
* @return
*/
public static boolean checkIsInPolygon(Point checkPoint, List<Point> areaAround) {
Point2D.Double point = new Point2D.Double(checkPoint.getLat(), checkPoint.getLng());
List<Point2D.Double> pointList = new ArrayList<>();
for (Point everyPoint : areaAround) {
Point2D.Double polygonPoint = new Point2D.Double(everyPoint.getLat(), everyPoint.getLng());
pointList.add(polygonPoint);
}
return checkIsPtInPoly(point, pointList);
}
/**
* 判断点是否在多边形内,如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
*
* @param point 检测点
* @param pts 多边形的顶点
* @return 点在多边形内返回true, 否则返回false
*/
public static boolean checkIsPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
int N = pts.size();
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
boolean boundOrVertex = true;
//cross points count of x
int intersectCount = 0;
//浮点类型计算时候与0比较时候的容差
double precision = 2e-10;
//neighbour bound vertices
Point2D.Double p1, p2;
//当前点
Point2D.Double p = point;
//left vertex
p1 = pts.get(0);
//check all rays
for (int i = 1; i <= N; ++i) {
if (p.equals(p1)) {
//p is an vertex
return boundOrVertex;
}
//right vertex
p2 = pts.get(i % N);
//ray is outside of our interests
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
p1 = p2;
//next ray left point
continue;
}
//ray is crossing over by the algorithm (common part of)
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
//x is before of ray
if (p.y <= Math.max(p1.y, p2.y)) {
//overlies on a horizontal ray
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
return boundOrVertex;
}
//ray is vertical
if (p1.y == p2.y) {
//overlies on a vertical ray
if (p1.y == p.y) {
return boundOrVertex;
//before ray
} else {
++intersectCount;
}
} else {
//cross point on the left side
//cross point of y
double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
//overlies on a ray
if (Math.abs(p.y - xinters) < precision) {
return boundOrVertex;
}
//before ray
if (p.y < xinters) {
++intersectCount;
}
}
}
} else {
//special case when ray is crossing through the vertex
//p crossing over p2
if (p.x == p2.x && p.y <= p2.y) {
//next vertex
Point2D.Double p3 = pts.get((i + 1) % N);
//p.x lies between p1.x & p3.x
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
++intersectCount;
} else {
intersectCount += 2;
}
}
}
//next ray left point
p1 = p2;
}
//偶数在多边形外
if (intersectCount % 2 == 0) {
return false;
} else {
//奇数在多边形内
return true;
}
}
public static void main(def a){
new PointUtil().run()
}
def run(){
def q = new BasicDBObject()
q.put("name", "淞南")
def obj = BUSINESS_RANGE_COLL.findOne(q)
if(obj){
def pointStr = obj.border
if(pointStr){
def pointArrs = pointStr.split(";")
def list = []
for(def s : pointArrs){
def arr = s.split(",")
list << new Point(arr[0].toDouble(), arr[1].toDouble())
}
println checkIsInPolygon(new Point(121.491478,31.327947), list)
}
}
}
}
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
def findCrawlData(){
def q = new BasicDBObject()
// q.put("city","福州")
// q.put("source", "安居客")
q.put("city","福州")
q.put("source", "城市房产")
q.put("s_date",202012)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = "anjuke.com/"
def prefix = ".cityhouse.cn/ha/"
def keyList =new HashSet()
for(def db : dbcursor){
def url = null
@@ -77,11 +77,11 @@ class CrawlDistrictIdSelect {
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com"))
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
@@ -92,7 +92,7 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.root = "cityhouse.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
@@ -1,7 +1,35 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import java.text.SimpleDateFormat
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date(1617211269868);
print dateFormat.format(date);
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
//print dateFormat.format(date);
DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
ExcelUtils.modify(path,{d->
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
d['G'] = districts.size()
def q = new BasicDBObject()
q.put("city", d['B'])
q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
def obj = CITY_INFO_CONN.findOne(q);
if(obj){
d['H'] = obj.price_coverage*100
}
})
@@ -1,6 +1,7 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash
@@ -13,8 +14,9 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客"]
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["zhuge.com":"诸葛找房"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -39,6 +41,7 @@ class SynchDistrictPrice {
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")));
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
@@ -921,13 +921,13 @@ class DateUtils {
public static String getRandomDate(def startDate, def endDate){
Date randomDate = randomDate(startDate, endDate);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
String result = format.format(randomDate);
return result;
}
static main(args){
println getRandomDate("2021-08-01","2021-08-31")
// nowFirstDay();
// nowLastDay();
/*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
@@ -955,8 +955,9 @@ class DateUtils {
// Date randomDate = randomDate("2010-09-20", "2011-09-22");
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
// String result = format.format(randomDate);
System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
// System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
println DateUtils.getDayOfWeek(new Date())
}
@@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def city = "À³Îß"
def destCity = "¼ÃÄÏ"
def region = ""
def destRegion = "À³Îß"
def city = "ÂåÑô"
def destCity = "ÂåÑô"
def region = "¼ªÀû"
def destRegion = "ÃϽò"