无经纬度不同步

This commit is contained in:
2021-08-19 19:50:31 +08:00
parent 1ea267b81f
commit e773ca283a
25 changed files with 1158 additions and 173 deletions
@@ -33,7 +33,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
for(def city : citys){ for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"]) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def rows = createDatas(cityCodeDatas.get(city), dists) def rows = createDatas(cityCodeDatas.get(city), dists)
rets.addAll(rows) rets.addAll(rows)
println city +" "+rows.size() println city +" "+rows.size()
@@ -390,7 +390,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap){ def getCityData(def city, def statMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","build_type","build_structure", "house_property","property_life"]) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList()); def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size()) final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -64,14 +64,14 @@ class BigdataForsaleBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = ["陵水"]//CcbCommonUtils.citys; def citys = ["陵水","文昌","保亭"]//CcbCommonUtils.citys;
def index = 0 def index = 0
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){ for(def city : citys){
index++ index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city)) // def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
} }
} }
@@ -0,0 +1,195 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 建行数据poi同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataPoiBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
runBatch(new BigdataPoiBatch(), args);
}
@Override
public void process(Object args) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def rets = []
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
rets.addAll(rows3)
//
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt"
}
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "txt 生成"
// def headers1 = ["城市ID","城市","行政区ID","区域名称","经度","纬度","创建时间","来源公司"]
// def regionCenterDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\行政区中心点经纬度.xlsx", 0,1)
// def regionRets = createRegionCenters(regionCenterDatas)
// FileUtils.batchCreateFile(regionRets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\region_center.txt", headers1, "utf-8", false);
// println "行政区域中心txt 生成"
}
def createRegionCenters(def datas){
def rets = []
for(def d : datas){
def ret = []
ret << d['A'].toInteger().toString()
ret << d['B']
ret << d['C'].toInteger().toString()
ret << d['D']
def loactionStr = d['E']
if(loactionStr){
def arrs = loactionStr.split(",")
def loactionMap = GeoUtils.gcjToGps84(arrs[1].toDouble(), arrs[0].toDouble())
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
}else{
ret << ""
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
return rets;
}
def createDatas(citys, cityCodeDataMap, datas, type){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def cityDistMap = CcbCommonUtils.findAllCityDistrictCodeValue()
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(citys.contains(d['D'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){
rets << ret
}
}
println totalCount.get()+" "+ d['D']+" "+d['F']
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['D'])
ret << citycode
def distCodeMap = cityDistMap.get(d['D'])
if(d['E']){
def distObj = distCodeMap.get(d['E'])
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << d['A']
ret << d['F']
ret << type
if(type=="地铁"){
ret << "cz125"
}else if(type=="公交"){
ret << "cz123"
}else if(type=="医院"){
ret << "hp111"
}else if(type == "银行"){
ret << "se118"
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['B'].contains("141202")){
ret << "ed138"
}else if(d['B'].contains("141203")){
ret << "ed114"
}
}
if(d['I']){
def lat = d['I'].toDouble()
def lng = d['H'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
return ret;
}
}
@@ -12,7 +12,8 @@ import com.mongodb.DBObject
class CcbCommonUtils { class CcbCommonUtils {
public static def citys = [ "扬州","上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] //
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"] // public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -47,6 +48,21 @@ class CcbCommonUtils {
return map; return map;
} }
public static def findAllCityDistrictCodeValue(city){
def q = new BasicDBObject()
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
def distMap = map.get(db.fangjia_city)
if(!distMap){
distMap = [:]
map.put(db.fangjia_city, distMap)
}
distMap.put(db.fangjia_district, db)
}
return map;
}
public static def findPersonPrice(city, date){ public static def findPersonPrice(city, date){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
@@ -170,7 +186,7 @@ class CcbCommonUtils {
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = [] def rets = []
for(def d : dbsour){ for(def d : dbsour){
if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){ if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.address){ if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", ""); d.address = d.address.replaceAll("\r|\n|\\s", "");
} }
@@ -27,7 +27,7 @@ class StagingDistrictBatch extends BaseBatch{
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
def rows = [] def rows = []
for(def city : citys){ for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","master_district_name", "c_date", "m_date"]) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
def fenQiMap = getFenQiMap(dists); def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap) def rets = createCityData(dists, fenQiMap)
rows.addAll(rets) rows.addAll(rets)
@@ -22,10 +22,6 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
//String str="\${ }";
//String regex="\\\$\\{.*\\}";
//Matcher m=Pattern.compile(regex).matcher(str);
//System.out.println(m.matches());
@@ -63,7 +59,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:] //def dealMap = [:]
//for(def d : datas){ //for(def d : datas){
@@ -79,48 +76,58 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
// dealMap.put(d['A'], map) // dealMap.put(d['A'], map)
//} //}
//def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//ExcelUtils.multiThreadingModify(fileName,{d-> //小区量
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def city = d['B'] // def city = d['B']
// if(city){ // if (city) {
// def cityid = cityCodeDatas.get(city) // def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance) // def total = findCount(cityid, sqlInstance)
//// d['I'] = total // d['K'] = total
//// def a = findEstateType(cityid, sqlInstance)
////
//// d['AA'] = total
//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
////
//// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
////
//// def c = findFiledCount(cityid, 'address', sqlInstance)
//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
////
//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
////
//// def e = findFiledCount(cityid, 'landarea', sqlInstance)
//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
////
//// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
////
//// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
////
//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
////
//// def i = findFiledCount(cityid, 'latitude', sqlInstance)
//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
////
//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
//
// println d
// } // }
//},0) //},0)
//小区字段覆盖率
ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['A']
if(city){
def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance)
d['AA'] = total
def a = findEstateType(cityid, sqlInstance)
d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
def b = findFiledCount(cityid, 'buildyear', sqlInstance)
d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
def c = findFiledCount(cityid, 'address', sqlInstance)
d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
def e = findFiledCount(cityid, 'landarea', sqlInstance)
d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
def f = findFiledCount(cityid, 'buildarea', sqlInstance)
d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
def g = findFiledCount(cityid, 'plotratio', sqlInstance)
d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
def i = findFiledCount(cityid, 'latitude', sqlInstance)
d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
println d
}
},1)
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx" //def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
//def dealMap = findDealCount(sqlInstance) //def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d-> //ExcelUtils.multiThreadingModify(fileName,{d->
@@ -206,64 +213,64 @@ def findFiledCount(cityid, field, sqlInstance){
} }
def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" //def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
def updateIndex = 0 //def updateIndex = 0
def delIndex = 0 //def delIndex = 0
def cityMap = [:] //def cityMap = [:]
ExcelUtils.multiThreadingModify(fileName,{d-> //ExcelUtils.multiThreadingModify(fileName,{d->
if(!d['R']){ // if(!d['R']){
//
def obj = CommonDatautils.findDistrictById(d['D']) // def obj = CommonDatautils.findDistrictById(d['D'])
if(obj){ // if(obj){
def ranges = cityMap.get(obj.city) // def ranges = cityMap.get(obj.city)
if(!ranges){ // if(!ranges){
def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) // def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
ranges = [] // ranges = []
for(def r : dbcursor){ // for(def r : dbcursor){
ranges << r // ranges << r
} // }
cityMap.put(obj.city, ranges) // cityMap.put(obj.city, ranges)
} // }
//
if(ranges){ // if(ranges){
def lng = obj.lng/10000000000.0 // def lng = obj.lng/10000000000.0
def lat = obj.lat/10000000000.0 // def lat = obj.lat/10000000000.0
//
def region = null // def region = null
for(def r : ranges){ // for(def r : ranges){
try{ // try{
def flag = isInArea(lng, lat, r.polyline) // def flag = isInArea(lng, lat, r.polyline)
if(flag){ // if(flag){
def regionStr = r.region // def regionStr = r.region
region = RegionData.formatRegion(obj.city, regionStr) // region = RegionData.formatRegion(obj.city, regionStr)
break // break
} // }
}catch(Exception e){ // }catch(Exception e){
println e.getMessage() // println e.getMessage()
} // }
//
} // }
if(region && region!=obj.region){ // if(region && region!=obj.region){
updateIndex++ // updateIndex++
DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) // DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name // println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
}else if(region && region==obj.region){ // }else if(region && region==obj.region){
println "[same] "+obj.city +" "+obj.name // println "[same] "+obj.city +" "+obj.name
}else{ // }else{
DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) // DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
delIndex++ // delIndex++
println delIndex +" [delete] "+obj.city +" "+obj.name // println delIndex +" [delete] "+obj.city +" "+obj.name
} // }
}else{ // }else{
println obj.city +" 没有此城市边框信息" // println obj.city +" 没有此城市边框信息"
} // }
//
}else{ // }else{
println d['D']+" 没找到该小区" // println d['D']+" 没找到该小区"
} // }
//
} // }
},0); //},0);
@@ -38,7 +38,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2"]) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city) def thirdPriceMap = findThirdPartyPrice(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
rows.addAll(rets) rows.addAll(rets)
@@ -180,58 +180,58 @@ class ThirdPartyPriceBatch extends BaseBatch {
dataMap.put("id", db._id.toString()) dataMap.put("id", db._id.toString())
map.put(key, dataMap) map.put(key, dataMap)
} }
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){ if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price) dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
} }
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){ if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price) dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
} }
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){ if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
} }
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){ if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
dataMap.put("soufang_esf_price", db.soufang_esf_price) dataMap.put("soufang_esf_price", db.soufang_esf_price)
} }
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price){ if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price) dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
} }
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price){ if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
} }
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price){ if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price) dataMap.put("maitian_esf_price", db.maitian_esf_price)
} }
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price){ if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price) dataMap.put("beike_esf_price", db.beike_esf_price)
} }
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price")){ if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
dataMap.put("365house_esf_price", db.get("365house_esf_price")) dataMap.put("365house_esf_price", db.get("365house_esf_price"))
} }
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price){ if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
} }
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price){ if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
} }
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price){ if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
dataMap.put("qfang_esf_price", db.qfang_esf_price) dataMap.put("qfang_esf_price", db.qfang_esf_price)
} }
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price){ if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
dataMap.put("fangtan_esf_price", db.fangtan_esf_price) dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
} }
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price){ if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price) dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
} }
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price")){ if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
} }
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ // if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
@@ -25,6 +25,7 @@ class TradeStatLog {
def task = new TradeStatLog() def task = new TradeStatLog()
//task.run() //task.run()
task.thirdPartyStat() task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
} }
@@ -88,7 +89,7 @@ class TradeStatLog {
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"] rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"]
for(def city : citys){ for(def city : citys){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat"]); def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat","lng"]);
def m = getThirdPartyPricePrecent(city) def m = getThirdPartyPricePrecent(city)
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14] rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
println rets.get(rets.size()-1) println rets.get(rets.size()-1)
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202106.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
@@ -1,6 +1,7 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import com.fangjia.core.Constants import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
@@ -29,7 +30,8 @@ class RegionPointCrawl {
def task = new RegionPointCrawl(); def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
def provinces = ["浙江省"] //def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
for(def p : provinces){ for(def p : provinces){
task.crawlCityAndDistrictCode(p); task.crawlCityAndDistrictCode(p);
task.run(p); task.run(p);
@@ -55,7 +57,13 @@ class RegionPointCrawl {
} }
println index +" "+cityMap.name+" "+m.name+" "+ret.size() println index +" "+cityMap.name+" "+m.name+" "+ret.size()
if(ret.size()>0){ if(ret.size()>0){
saveData(cityMap.name, m.name, m.ad_code , ret); def standardCity = RegionData.formatCity(cityMap.name)
if(standardCity){
saveData(standardCity, m.name, m.ad_code , ret);
}else{
println cityMap.name +" 没有标准化"
}
} }
} }
} }
@@ -6,6 +6,7 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
@@ -47,14 +48,13 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-06-01" static def statDate = "2021-07-01"
static main(args) { static main(args) {
def task = new YouDaDataParser(); def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo // task.newHouseparse();//友达一手房存入mongo
task.esfParser();//友达二手房存入mongo // task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es // task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
} }
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){ def esfParser(){
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"] def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"]
def datas = [] def datas = []
for(def fileName : files){ for(def fileName : files){
def rows = readDBF(fileName); def rows = readDBF(fileName);
@@ -308,10 +308,7 @@ class YouDaDataParser {
} }
); );
} }
latch.await() latch.await()
} }
def saveEsfHouseToDB(def data){ def saveEsfHouseToDB(def data){
@@ -367,15 +364,16 @@ class YouDaDataParser {
row.ydId = data[2].toInteger() row.ydId = data[2].toInteger()
row.fjCity = city row.fjCity = city
def map = matchDistrict(city, row.region, row.districtName) // def map = matchDistrict(city, row.region, row.districtName)
if(!map){ // if(!map){
map = matchDistrict(city, row.region, row.address) // map = matchDistrict(city, row.region, row.address)
} // }
def map = SeaweedDataMatchProcess.doMatch(city, row.region, row.districtName, row.address);
if(map){ if(map){
row.fjRegion = map.region row.fjRegion = map.region
row.fjName = map.name row.fjName = map.name
row.lat = map.lat row.lat = map.lat.toDouble()
row.lng = map.lng row.lng = map.lng.toDouble()
row.location = map.lat.toString()+","+map.lng.toString() row.location = map.lat.toString()+","+map.lng.toString()
def obj = CommonDatautils.findDistrictById(map.id.toString()); def obj = CommonDatautils.findDistrictById(map.id.toString());
@@ -7,16 +7,22 @@ import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject import com.mongodb.DBObject
import com.rabbitmq.client.AMQP import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
class AddDistrictFieldValue { class AddDistrictFieldValue {
@@ -30,18 +36,26 @@ class AddDistrictFieldValue {
} }
def run(){ def run(){
def citys = CcbCommonUtils.citys; def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){ // for(def city : citys){
// def rets = findAllDataByCity(city) // def rets = findAllDataByCity(city)
// match(rets) // match(rets)
// } // }
for(def city : citys){ //第三方均价数据填补
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) // for(def city : citys){
def rMap = getMatchDistrict(city) // def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
createResultValue(dists, rMap) // def rMap = getMatchDistrict(city)
} // createResultValue(dists, rMap)
// }
//
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
// addFinishDate(dists)
addFieldValueByHouse(dists)
}
} }
def match(def rets){ def match(def rets){
@@ -118,6 +132,35 @@ class AddDistrictFieldValue {
return rMap; return rMap;
} }
def addFinishDate(dists){
def updateIndex = 0
def total = dists.size()
for(def d : dists){
if(!d.finish_building_date){
def udata = new BasicDBObject()
def newValue = FiledStrand.processFinishBuildingDate(d.live_date)
if(!newValue && d.open_date){
def date = d.open_date
newValue = date.getYear()+1900
}
if(!newValue && d.build_date){
newValue = FiledStrand.processFinishBuildingDate(d.build_date)
}
if(newValue){
udata.put("finish_building_date", newValue)
}
if(udata.size()>0){
updateIndex++
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
def createResultValue(dists, rMap){ def createResultValue(dists, rMap){
def updateIndex = 0 def updateIndex = 0
def total = dists.size() def total = dists.size()
@@ -260,9 +303,163 @@ class AddDistrictFieldValue {
} }
} }
} }
return lastValue; return lastValue;
} }
def addFieldValueByHouse(def dists){
def updateIndex = new AtomicLong()
def totalCount = new AtomicLong()
def total = dists.size()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(d.estate_type2!="别墅" && (!d.finish_building_date || !d.build_type)){
def udata = new BasicDBObject()
def retMap = findHouses(d.city, d.region, "sell", d.name, d.alias)
if(retMap){
println retMap
if(!d.finish_building_date && retMap.containsKey("build_date")){
def dateMap = retMap.get("build_date")
if(dateMap){
def value = dateMap.keySet()[0]
if(value){
udata.put("finish_building_date", value)
}
}
}
if(!d.build_type && retMap.containsKey("height")){
def heightMap = retMap.get("height")
if(heightMap){
def heights = heightMap.keySet()
if(heights){
def max = heights.max()
if(max<=6){
udata.put("build_type", "多层")
}else if(max>6 && max<=11){
udata.put("build_type", "小高层")
}else if(max>11 && max<30){
udata.put("build_type", "高层")
}else if(max>30){
udata.put("build_type", "超高层")
}
}
}
}
}
if(udata.size()>0){
updateIndex.addAndGet(1)
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex.get() +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}else{
println totalCount.get() +" "+d.city +" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
latch.await(5, TimeUnit.MINUTES);
}
def findHouses(def city, def region, def cat, def name, def alias){
// QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
// must(QueryBuilders.termQuery("cat", cat)).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def restMap = [:]
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(seawater.build_date){
def newValue = FiledStrand.processFinishBuildingDate(seawater.build_date)
if(newValue){
def valueMap = restMap.get("build_date")
if(!valueMap){
valueMap = [:]
restMap.put("build_date", valueMap)
}
def count = valueMap.get(newValue)
if(!count){
valueMap.put(newValue, 1)
}else{
valueMap.put(newValue, count + 1)
}
}
}
if(seawater.height){
def hightMap = restMap.get("height")
if(!hightMap){
hightMap = [:]
restMap.put("height", hightMap)
}
def count = hightMap.get(seawater.height)
if(!count){
count = 1
hightMap.put(seawater.height, count)
}else{
hightMap.put(seawater.height, count + 1)
}
}
}
}
if(restMap){
def dateMap = restMap.get("build_date")
if(dateMap){
dateMap = dateMap.sort {a, b -> b.value <=> a.value}
restMap.put("build_date", dateMap)
}
}
return restMap;
}
} }
@@ -24,7 +24,7 @@ class DataProcess {
static main(args) { static main(args) {
//建行信用卡城市(临汾--晋城).xlsx //建行信用卡城市(临汾--晋城).xlsx
def filePath = "D:\\数据\\小区\\地图\\张家口.xlsx" def filePath = "D:\\数据\\小区\\地图\\兰州.xlsx"
def task = new DataProcess(); def task = new DataProcess();
// task.batchSaveDataToDB(filePath) // task.batchSaveDataToDB(filePath)
task.mapBatchSaveDataToDB(filePath) task.mapBatchSaveDataToDB(filePath)
@@ -121,7 +121,7 @@ class DataProcess {
if(datas){ if(datas){
def keys = [] def keys = []
for(def data : datas){//20,21,25, for(def data : datas){//20,21,25,
if(data['I'] && [20,21,25,26,27].contains(data['I'].toString().toDouble().toInteger())){ if(data['I'] && [20,21,26,27].contains(data['I'].toString().toDouble().toInteger())){
def key = data['A']+"##"+data['B']+"##"+data['H'] def key = data['A']+"##"+data['B']+"##"+data['H']
if(!keys.contains(key) && data['H'] && data['H'].size()>2){ if(!keys.contains(key) && data['H'] && data['H'].size()>2){
keys << key keys << key
@@ -5,15 +5,22 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import com.mongodb.BasicDBObject
import java.util.regex.Pattern import java.util.regex.Pattern
/***
* 地图小区处理
*/
class MapDistrictProcess { class MapDistrictProcess {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) { static main(args) {
def task = new MapDistrictProcess(); def task = new MapDistrictProcess();
task.run() task.run()
@@ -28,17 +35,18 @@ class MapDistrictProcess {
def run(){ def run(){
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"] // def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
def files = ["青岛.xlsx","沈阳.xlsx","哈尔滨.xlsx","张家口.xlsx","大连.xlsx"] //"乌鲁木齐.xlsx","烟台.xlsx","保定.xlsx","贵阳.xlsx","湖州.xlsx","惠州.xlsx","吉林.xlsx","遵义.xlsx","宜昌.xlsx","芜湖.xlsx","兰州.xlsx"
def files = ["兰州.xlsx"]
for(def fileName : files){ for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName; def filePath = "D:\\数据\\小区\\地图\\"+fileName;
typeFilter(filePath) typeFilter(filePath)
nameStandard(filePath); nameStandard(filePath);
dataFilter(filePath) dataFilter(filePath)
houseCalculate(filePath); houseCalculate(filePath);
////// //统计是否存在多个区域 //统计是否存在多个区域
matchDistDatas(filePath) matchDistDatas(filePath)
moreRegionCalculate(filePath) moreRegionCalculate(filePath)
//
addFlag(filePath) addFlag(filePath)
mapMatch(filePath); mapMatch(filePath);
baiduTypeFilter(filePath); baiduTypeFilter(filePath);
@@ -50,7 +58,8 @@ class MapDistrictProcess {
addFeature(filePath) addFeature(filePath)
houseMatch(filePath); houseMatch(filePath);
// nameScore(filePath); nameScore(filePath);
thirdPartyData(filePath)
} }
} }
@@ -212,7 +221,7 @@ class MapDistrictProcess {
def addFlag(def filePath){ def addFlag(def filePath){
def index = 0 def index = 0
ExcelUtils.modify(filePath, { row -> ExcelUtils.multiThreadingModify(filePath, { row ->
index++ index++
if(!row['I']) { if(!row['I']) {
//根据别名查找 //根据别名查找
@@ -438,6 +447,34 @@ class MapDistrictProcess {
},0) },0)
} }
def thirdPartyData(filePath){
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['I'] && [20,21,25,26,27].contains(row['I'].toString().toDouble().toInteger())){
def rets = findIsExistsOtherWebsite(row['A'], row['B'], row['H'])
if(rets){
row['W'] = rets.join(",")
}
}
},0)
}
def findIsExistsOtherWebsite(def city, def region, def name){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name", name)
def dbcursor = districtDetailColl.find(q)
def rets = []
for(def r : dbcursor){
if(!rets.contains(r.root)){
rets << r.root
}
}
return rets
}
def getRegionByLocation(lng, lat){ def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString()) def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
@@ -25,6 +25,9 @@ class SeaweedDataMatchProcess {
rMap.city = obj.city rMap.city = obj.city
rMap.region = obj.region rMap.region = obj.region
rMap.name = obj.name rMap.name = obj.name
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
rMap.id = obj._id.toString()
} }
} }
} }
@@ -0,0 +1,320 @@
package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
class AddDetailDistrict {
def detailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) {
def task = new AddDetailDistrict();
// task.run();
// task.delRepeatDist
task.addEstateType()
}
def addEstateType(){
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","estate_type2"])
for(def d : districts) {
if(!d.estate_type2){
def estateType2 = "普通住宅"
if(d.name.contains("别墅")){
estateType2 = "别墅"
}else if(d.name.contains("商住")){
estateType2 = "商住"
}
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("estate_type2", estateType2).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
println city
}
}
def delRepeatDist(){
def citys = CcbCommonUtils.citys
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
def index = 0
for(def d : districts){
index++
def key = d.city +"##"+d.region+"##"+d.name
if(!keys.contains(key)){
keys << key
}else{
println index +" "+d.region+" "+d.name
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("status", 1).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
}
}
def run() {
def citys = ["沈阳", "哈尔滨", "吉林", "遵义", "淄博", "西宁", "青岛", "大连", "宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
for (def city : citys) {
def rets = findAllDataByCity(city)
def rMap = getMatchDistrict(city)
toDB(rets, rMap)
}
}
def toDB(def rets, def rMap){
def saveDistKeys = []
def total = rets.size()
def index = 0
def addIndex = 0
for(def d : rets){
index++
def newRegion = RegionData.formatRegion(d.city, d.region)
def name = NameProcess.nameStandard(d.city, newRegion, d.name)
if(!name){
continue
}
def flag = NameProcess.filterName(name)
def key = null
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
if(flag && !saveDistKeys.contains(key)){
def gdData = FangjiaClientUtils.getMapData(d.city, newRegion, name, d.address,"", "高德")
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(gdData){
if(!newRegion){
newRegion = RegionData.formatRegion(d.city, gdData.district)
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
}
}
if(key && gdData && gdData.lng){
def matchMap = SeaweedDataMatchProcess.doMatch(d.city, newRegion, name, d.address);
if(!matchMap){
addIndex++
if(gdData.lng){
def gmap = GeocodeUtils.getGaodeToBaidu(gdData.lng.toDouble(), gdData.lat.toDouble())
lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
lng = (gdData.lng.toDouble()* 10000000000).longValue()
lat = (gdData.lat.toDouble()* 10000000000).longValue()
}
def newAddress = DataUtil.formateAddress(d.city, d.address)
def udata = createFieldValue(d, rMap)
udata.put("lng", lng)
udata.put("lat", lat)
println index+"/${addIndex}/${total} "+d.city+" "+newRegion+" "+name+" "+d.root+" "+udata
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", name)
udata.put("address", newAddress)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("status", 0)
udata.put("cat", "district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
DbUtils.seaweed.save(udata)
saveDistKeys << key
}
}
// println d.city+" "+ d.region+" "+d.name
}
}
}
def createFieldValue(d, rMap){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
def udata = new BasicDBObject()
if(sourceMap){
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
udata.put("all_area", lastValue)
}
}
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
udata.put("build_area", lastValue)
}
}
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
udata.put("virescence_percent", lastValue)
}
}
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
udata.put("cubage_percent", lastValue)
}
}
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
udata.put("finish_building_date", lastValue)
}
}
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
udata.put("household_count", lastValue)
}
}
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
udata.put("car_site_count", lastValue)
}
}
}
return udata
}
def getValuesByField(sourceMap, field, estateType){
def map = [:]
sourceMap.each{k,v->
def oldValue = v.get(field);
if(oldValue){
def newValue = null;
if(["address"].contains(field)){
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
}
if(["all_area","build_area"].contains(field)){
newValue = FiledStrand.processArea(oldValue)
}
if(["build_date"].contains(field)){
newValue = FiledStrand.processFinishBuildingDate(oldValue)
}
if(["house_count"].contains(field)){
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
}
if(["greening_rate"].contains(field)){
newValue = FiledStrand.processVirescencePercent(oldValue)
}
if(["volumetric_rate"].contains(field)){
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
}
if(["car_site_desc","car_site"].contains(field)){
newValue = FiledStrand.processCarSite(oldValue)
}
if(newValue){
def count = map.get(newValue)
if(!count){
map.put(newValue, 1)
}else{
map.put(newValue, count+1)
}
v.put(field, newValue)
}
}
}
def lastValue = null
if(map && map.size()>0){
map = map.sort {a, b -> b.value <=> a.value}
def values = map.values()
if(values.size()==1){
lastValue = map.keySet()[0]
}else{
def count1 = map.values()[0]
def count2 = map.values()[1]
if(count1 > count2){
lastValue = map.keySet()[0]
}else{
def rets = [map.keySet()[0],map.keySet()[1]]
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
for(def root : roots){
def data = sourceMap.get(root)
if(data){
def value = data.get(field)
if(rets.contains(value)){
lastValue = value
break
}
}
}
}
}
}
return lastValue;
}
def getMatchDistrict(def city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("match_name", new BasicDBObject('$exists', true))
def dbcursor = detailColl.find(q)
def rMap = [:]
for(def d : dbcursor){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
if(!sourceMap){
sourceMap = [:]
rMap.put(key, sourceMap)
}
def value = sourceMap.get(d.root)
if(!value){
sourceMap.put(d.root, d)
}
}
return rMap;
}
def findAllDataByCity(def city){
def q = new BasicDBObject("city", city);
q.put("match_name", new BasicDBObject('$exists', false))
q.put("avg_price", new BasicDBObject('$gt', 0))
def dbcursor = detailColl.find(q).batchSize(2000)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -1,8 +1,9 @@
package com.fangjia.script.gj package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.datastore.FieldValidatorUtils import com.fangjia.script.datastore.FieldValidatorUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
@@ -0,0 +1,27 @@
package com.fangjia.script.gj.location
class Point {
Double lat;
Double lng;
public Point(Double lng, Double lat){
this.lat = lat
this.lng = lng
}
Double getLat() {
return lat
}
void setLat(Double lat) {
this.lat = lat
}
Double getLng() {
return lng
}
void setLng(Double lng) {
this.lng = lng
}
}
@@ -0,0 +1,143 @@
package com.fangjia.script.gj.location
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.awt.geom.Point2D
class PointUtil {
public static final DBCollection BUSINESS_RANGE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border", "mongo.db.fangjia.com")
/**
* 判断当前位置是否在多边形区域内
*
* @param checkPoint 当前经纬度
* @param areaAround 围栏范围
* @return
*/
public static boolean checkIsInPolygon(Point checkPoint, List<Point> areaAround) {
Point2D.Double point = new Point2D.Double(checkPoint.getLat(), checkPoint.getLng());
List<Point2D.Double> pointList = new ArrayList<>();
for (Point everyPoint : areaAround) {
Point2D.Double polygonPoint = new Point2D.Double(everyPoint.getLat(), everyPoint.getLng());
pointList.add(polygonPoint);
}
return checkIsPtInPoly(point, pointList);
}
/**
* 判断点是否在多边形内,如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
*
* @param point 检测点
* @param pts 多边形的顶点
* @return 点在多边形内返回true, 否则返回false
*/
public static boolean checkIsPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
int N = pts.size();
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
boolean boundOrVertex = true;
//cross points count of x
int intersectCount = 0;
//浮点类型计算时候与0比较时候的容差
double precision = 2e-10;
//neighbour bound vertices
Point2D.Double p1, p2;
//当前点
Point2D.Double p = point;
//left vertex
p1 = pts.get(0);
//check all rays
for (int i = 1; i <= N; ++i) {
if (p.equals(p1)) {
//p is an vertex
return boundOrVertex;
}
//right vertex
p2 = pts.get(i % N);
//ray is outside of our interests
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
p1 = p2;
//next ray left point
continue;
}
//ray is crossing over by the algorithm (common part of)
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
//x is before of ray
if (p.y <= Math.max(p1.y, p2.y)) {
//overlies on a horizontal ray
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
return boundOrVertex;
}
//ray is vertical
if (p1.y == p2.y) {
//overlies on a vertical ray
if (p1.y == p.y) {
return boundOrVertex;
//before ray
} else {
++intersectCount;
}
} else {
//cross point on the left side
//cross point of y
double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
//overlies on a ray
if (Math.abs(p.y - xinters) < precision) {
return boundOrVertex;
}
//before ray
if (p.y < xinters) {
++intersectCount;
}
}
}
} else {
//special case when ray is crossing through the vertex
//p crossing over p2
if (p.x == p2.x && p.y <= p2.y) {
//next vertex
Point2D.Double p3 = pts.get((i + 1) % N);
//p.x lies between p1.x & p3.x
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
++intersectCount;
} else {
intersectCount += 2;
}
}
}
//next ray left point
p1 = p2;
}
//偶数在多边形外
if (intersectCount % 2 == 0) {
return false;
} else {
//奇数在多边形内
return true;
}
}
public static void main(def a){
new PointUtil().run()
}
def run(){
def q = new BasicDBObject()
q.put("name", "淞南")
def obj = BUSINESS_RANGE_COLL.findOne(q)
if(obj){
def pointStr = obj.border
if(pointStr){
def pointArrs = pointStr.split(";")
def list = []
for(def s : pointArrs){
def arr = s.split(",")
list << new Point(arr[0].toDouble(), arr[1].toDouble())
}
println checkIsInPolygon(new Point(121.491478,31.327947), list)
}
}
}
}
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
def findCrawlData(){ def findCrawlData(){
def q = new BasicDBObject() def q = new BasicDBObject()
// q.put("city","福州") q.put("city","福州")
// q.put("source", "安居客") q.put("source", "城市房产")
q.put("s_date",202012) q.put("s_date",202012)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:] def retMap = [:]
def prefix = "anjuke.com/" def prefix = ".cityhouse.cn/ha/"
def keyList =new HashSet() def keyList =new HashSet()
for(def db : dbcursor){ for(def db : dbcursor){
def url = null def url = null
@@ -77,11 +77,11 @@ class CrawlDistrictIdSelect {
url = db.url url = db.url
} }
if(url && url.contains(prefix) ){ if(url && url.contains(prefix) ){
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1) def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
if(district_id=="0"){ if(district_id=="0"){
continue continue
} }
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com")) def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
def key = shortCity +"#"+district_id def key = shortCity +"#"+district_id
def ids = retMap.get(key) def ids = retMap.get(key)
@@ -92,7 +92,7 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){ if(!keyList.contains(key)){
def map = new BasicDBObject() def map = new BasicDBObject()
map.root = "anjuke.com" map.root = "cityhouse.com"
map.city_short= shortCity map.city_short= shortCity
map.city = db.city map.city = db.city
map.region = db.region map.region = db.region
@@ -1,7 +1,35 @@
package com.fangjia.script.gj.test package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date(1617211269868); //Date date = new Date(1617211269868);
print dateFormat.format(date); //print dateFormat.format(date);
DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
ExcelUtils.modify(path,{d->
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
d['G'] = districts.size()
def q = new BasicDBObject()
q.put("city", d['B'])
q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
def obj = CITY_INFO_CONN.findOne(q);
if(obj){
d['H'] = obj.price_coverage*100
}
})
@@ -1,6 +1,7 @@
package com.fangjia.script.thirdPriceCrawl package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash import org.apache.solr.common.util.Hash
@@ -13,8 +14,9 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth() def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客"] // "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["zhuge.com":"诸葛找房"]
static main(args) { static main(args) {
def task = new SynchDistrictPrice() def task = new SynchDistrictPrice()
task.run() task.run()
@@ -39,6 +41,7 @@ class SynchDistrictPrice {
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
q.put("root", root) q.put("root", root)
q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")));
def keys = new BasicDBObject() def keys = new BasicDBObject()
keys.put("city", 1) keys.put("city", 1)
keys.put("region", 1) keys.put("region", 1)
@@ -921,13 +921,13 @@ class DateUtils {
public static String getRandomDate(def startDate, def endDate){ public static String getRandomDate(def startDate, def endDate){
Date randomDate = randomDate(startDate, endDate); Date randomDate = randomDate(startDate, endDate);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
String result = format.format(randomDate); String result = format.format(randomDate);
return result; return result;
} }
static main(args){ static main(args){
println getRandomDate("2021-08-01","2021-08-31")
// nowFirstDay(); // nowFirstDay();
// nowLastDay(); // nowLastDay();
/*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd"); /*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
@@ -955,8 +955,9 @@ class DateUtils {
// Date randomDate = randomDate("2010-09-20", "2011-09-22"); // Date randomDate = randomDate("2010-09-20", "2011-09-22");
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); // SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
// String result = format.format(randomDate); // String result = format.format(randomDate);
System.out.println(getRandomDate("2010-09-20", "2011-09-22")); // System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
println DateUtils.getDayOfWeek(new Date())
} }
@@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils import com.fangjia.core.util.Mongo3Utils
def city = "À³Îß" def city = "ÂåÑô"
def destCity = "¼ÃÄÏ" def destCity = "ÂåÑô"
def region = "" def region = "¼ªÀû"
def destRegion = "À³Îß" def destRegion = "ÃϽò"