无经纬度不同步
This commit is contained in:
+1
-1
@@ -33,7 +33,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
|
||||
def rows = createDatas(cityCodeDatas.get(city), dists)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
|
||||
+1
-1
@@ -390,7 +390,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def getCityData(def city, def statMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","build_type","build_structure", "house_property","property_life"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
+2
-2
@@ -64,14 +64,14 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["陵水"]//CcbCommonUtils.citys;
|
||||
def citys = ["陵水","文昌","保亭"]//CcbCommonUtils.citys;
|
||||
def index = 0
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
|
||||
}
|
||||
}
|
||||
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 建行数据poi同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataPoiBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataPoiBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
|
||||
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
||||
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
||||
|
||||
def rets = []
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
rets.addAll(rows1)
|
||||
|
||||
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
rets.addAll(rows2)
|
||||
//
|
||||
def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
rets.addAll(rows3)
|
||||
//
|
||||
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
rets.addAll(rows4)
|
||||
|
||||
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
rets.addAll(rows5)
|
||||
|
||||
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
rets.addAll(rows6)
|
||||
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt"
|
||||
}
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
|
||||
// def headers1 = ["城市ID","城市","行政区ID","区域名称","经度","纬度","创建时间","来源公司"]
|
||||
// def regionCenterDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\行政区中心点经纬度.xlsx", 0,1)
|
||||
// def regionRets = createRegionCenters(regionCenterDatas)
|
||||
// FileUtils.batchCreateFile(regionRets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\region_center.txt", headers1, "utf-8", false);
|
||||
// println "行政区域中心txt 生成"
|
||||
|
||||
}
|
||||
|
||||
def createRegionCenters(def datas){
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def ret = []
|
||||
ret << d['A'].toInteger().toString()
|
||||
ret << d['B']
|
||||
ret << d['C'].toInteger().toString()
|
||||
ret << d['D']
|
||||
def loactionStr = d['E']
|
||||
if(loactionStr){
|
||||
def arrs = loactionStr.split(",")
|
||||
def loactionMap = GeoUtils.gcjToGps84(arrs[1].toDouble(), arrs[0].toDouble())
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
return rets;
|
||||
|
||||
}
|
||||
|
||||
def createDatas(citys, cityCodeDataMap, datas, type){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
def cityDistMap = CcbCommonUtils.findAllCityDistrictCodeValue()
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(citys.contains(d['D'])){
|
||||
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
println totalCount.get()+" "+ d['D']+" "+d['F']
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addValue(d, cityCodeDataMap, cityDistMap, type){
|
||||
def ret = []
|
||||
def citycode = cityCodeDataMap.get(d['D'])
|
||||
ret << citycode
|
||||
def distCodeMap = cityDistMap.get(d['D'])
|
||||
if(d['E']){
|
||||
def distObj = distCodeMap.get(d['E'])
|
||||
if(distObj){
|
||||
ret << distObj.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << d['A']
|
||||
ret << d['F']
|
||||
ret << type
|
||||
|
||||
if(type=="地铁"){
|
||||
ret << "cz125"
|
||||
}else if(type=="公交"){
|
||||
ret << "cz123"
|
||||
}else if(type=="医院"){
|
||||
ret << "hp111"
|
||||
}else if(type == "银行"){
|
||||
ret << "se118"
|
||||
}else if(type =="丧葬服务"){
|
||||
ret << "ob101"
|
||||
}else if(type =="学校"){
|
||||
if(d['B'].contains("141202")){
|
||||
ret << "ed138"
|
||||
}else if(d['B'].contains("141203")){
|
||||
ret << "ed114"
|
||||
}
|
||||
}
|
||||
if(d['I']){
|
||||
def lat = d['I'].toDouble()
|
||||
def lng = d['H'].toDouble()
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
+18
-2
@@ -12,7 +12,8 @@ import com.mongodb.DBObject
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
public static def citys = [ "扬州","上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
|
||||
//
|
||||
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
|
||||
// public static def citys = ["武汉"]
|
||||
|
||||
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
|
||||
@@ -47,6 +48,21 @@ class CcbCommonUtils {
|
||||
return map;
|
||||
}
|
||||
|
||||
public static def findAllCityDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def distMap = map.get(db.fangjia_city)
|
||||
if(!distMap){
|
||||
distMap = [:]
|
||||
map.put(db.fangjia_city, distMap)
|
||||
}
|
||||
distMap.put(db.fangjia_district, db)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public static def findPersonPrice(city, date){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
@@ -170,7 +186,7 @@ class CcbCommonUtils {
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","master_district_name", "c_date", "m_date"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
|
||||
def fenQiMap = getFenQiMap(dists);
|
||||
def rets = createCityData(dists, fenQiMap)
|
||||
rows.addAll(rets)
|
||||
|
||||
+107
-100
@@ -22,10 +22,6 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
|
||||
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
//String str="\${ }";
|
||||
//String regex="\\\$\\{.*\\}";
|
||||
//Matcher m=Pattern.compile(regex).matcher(str);
|
||||
//System.out.println(m.matches());
|
||||
|
||||
|
||||
|
||||
@@ -63,7 +59,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
|
||||
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
|
||||
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx"
|
||||
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
//for(def d : datas){
|
||||
@@ -79,48 +76,58 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
// dealMap.put(d['A'], map)
|
||||
//}
|
||||
|
||||
//def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
//小区量
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def city = d['B']
|
||||
// if(city){
|
||||
// if (city) {
|
||||
// def cityid = cityCodeDatas.get(city)
|
||||
// def total = findCount(cityid, sqlInstance)
|
||||
//// d['I'] = total
|
||||
//// def a = findEstateType(cityid, sqlInstance)
|
||||
////
|
||||
//// d['AA'] = total
|
||||
//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
|
||||
////
|
||||
//// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
|
||||
//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
|
||||
////
|
||||
//// def c = findFiledCount(cityid, 'address', sqlInstance)
|
||||
//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
|
||||
////
|
||||
//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
|
||||
//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
|
||||
////
|
||||
//// def e = findFiledCount(cityid, 'landarea', sqlInstance)
|
||||
//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
|
||||
////
|
||||
//// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
|
||||
//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
|
||||
////
|
||||
//// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
|
||||
//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
|
||||
////
|
||||
//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
|
||||
//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
|
||||
////
|
||||
//// def i = findFiledCount(cityid, 'latitude', sqlInstance)
|
||||
//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
|
||||
////
|
||||
//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
|
||||
//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
|
||||
//
|
||||
// println d
|
||||
// d['K'] = total
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//小区字段覆盖率
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def city = d['A']
|
||||
if(city){
|
||||
def cityid = cityCodeDatas.get(city)
|
||||
def total = findCount(cityid, sqlInstance)
|
||||
d['AA'] = total
|
||||
|
||||
def a = findEstateType(cityid, sqlInstance)
|
||||
d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
|
||||
|
||||
def b = findFiledCount(cityid, 'buildyear', sqlInstance)
|
||||
d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
|
||||
|
||||
def c = findFiledCount(cityid, 'address', sqlInstance)
|
||||
d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
|
||||
|
||||
def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
|
||||
d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
|
||||
|
||||
def e = findFiledCount(cityid, 'landarea', sqlInstance)
|
||||
d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
|
||||
|
||||
def f = findFiledCount(cityid, 'buildarea', sqlInstance)
|
||||
d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
|
||||
|
||||
def g = findFiledCount(cityid, 'plotratio', sqlInstance)
|
||||
d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
|
||||
|
||||
def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
|
||||
d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
|
||||
|
||||
def i = findFiledCount(cityid, 'latitude', sqlInstance)
|
||||
d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
|
||||
|
||||
def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
|
||||
d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
|
||||
println d
|
||||
}
|
||||
},1)
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance)
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
@@ -206,64 +213,64 @@ def findFiledCount(cityid, field, sqlInstance){
|
||||
}
|
||||
|
||||
|
||||
def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
|
||||
def updateIndex = 0
|
||||
def delIndex = 0
|
||||
def cityMap = [:]
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(!d['R']){
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['D'])
|
||||
if(obj){
|
||||
def ranges = cityMap.get(obj.city)
|
||||
if(!ranges){
|
||||
def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
|
||||
ranges = []
|
||||
for(def r : dbcursor){
|
||||
ranges << r
|
||||
}
|
||||
cityMap.put(obj.city, ranges)
|
||||
}
|
||||
|
||||
if(ranges){
|
||||
def lng = obj.lng/10000000000.0
|
||||
def lat = obj.lat/10000000000.0
|
||||
|
||||
def region = null
|
||||
for(def r : ranges){
|
||||
try{
|
||||
def flag = isInArea(lng, lat, r.polyline)
|
||||
if(flag){
|
||||
def regionStr = r.region
|
||||
region = RegionData.formatRegion(obj.city, regionStr)
|
||||
break
|
||||
}
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
if(region && region!=obj.region){
|
||||
updateIndex++
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
|
||||
println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
|
||||
}else if(region && region==obj.region){
|
||||
println "[same] "+obj.city +" "+obj.name
|
||||
}else{
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
|
||||
delIndex++
|
||||
println delIndex +" [delete] "+obj.city +" "+obj.name
|
||||
}
|
||||
}else{
|
||||
println obj.city +" 没有此城市边框信息"
|
||||
}
|
||||
|
||||
}else{
|
||||
println d['D']+" 没找到该小区"
|
||||
}
|
||||
|
||||
}
|
||||
},0);
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
|
||||
//def updateIndex = 0
|
||||
//def delIndex = 0
|
||||
//def cityMap = [:]
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(!d['R']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['D'])
|
||||
// if(obj){
|
||||
// def ranges = cityMap.get(obj.city)
|
||||
// if(!ranges){
|
||||
// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
|
||||
// ranges = []
|
||||
// for(def r : dbcursor){
|
||||
// ranges << r
|
||||
// }
|
||||
// cityMap.put(obj.city, ranges)
|
||||
// }
|
||||
//
|
||||
// if(ranges){
|
||||
// def lng = obj.lng/10000000000.0
|
||||
// def lat = obj.lat/10000000000.0
|
||||
//
|
||||
// def region = null
|
||||
// for(def r : ranges){
|
||||
// try{
|
||||
// def flag = isInArea(lng, lat, r.polyline)
|
||||
// if(flag){
|
||||
// def regionStr = r.region
|
||||
// region = RegionData.formatRegion(obj.city, regionStr)
|
||||
// break
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
// println e.getMessage()
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// if(region && region!=obj.region){
|
||||
// updateIndex++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
|
||||
// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
|
||||
// }else if(region && region==obj.region){
|
||||
// println "[same] "+obj.city +" "+obj.name
|
||||
// }else{
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
|
||||
// delIndex++
|
||||
// println delIndex +" [delete] "+obj.city +" "+obj.name
|
||||
// }
|
||||
// }else{
|
||||
// println obj.city +" 没有此城市边框信息"
|
||||
// }
|
||||
//
|
||||
// }else{
|
||||
// println d['D']+" 没找到该小区"
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//},0);
|
||||
|
||||
|
||||
|
||||
|
||||
+16
-16
@@ -38,7 +38,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def thirdPriceMap = findThirdPartyPrice(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
|
||||
rows.addAll(rets)
|
||||
@@ -180,58 +180,58 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
dataMap.put("id", db._id.toString())
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){
|
||||
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){
|
||||
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
dataMap.put("soufang_esf_price", db.soufang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price){
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price){
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price){
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price){
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price")){
|
||||
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price){
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price){
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price){
|
||||
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
dataMap.put("qfang_esf_price", db.qfang_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price){
|
||||
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price){
|
||||
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price")){
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
|
||||
+2
-1
@@ -25,6 +25,7 @@ class TradeStatLog {
|
||||
def task = new TradeStatLog()
|
||||
//task.run()
|
||||
task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
|
||||
}
|
||||
|
||||
@@ -88,7 +89,7 @@ class TradeStatLog {
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"]
|
||||
for(def city : citys){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat","lng"]);
|
||||
def m = getThirdPartyPricePrecent(city)
|
||||
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
|
||||
println rets.get(rets.size()-1)
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202106.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
@@ -29,7 +30,8 @@ class RegionPointCrawl {
|
||||
def task = new RegionPointCrawl();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
def provinces = ["浙江省"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
@@ -55,7 +57,13 @@ class RegionPointCrawl {
|
||||
}
|
||||
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
|
||||
if(ret.size()>0){
|
||||
saveData(cityMap.name, m.name, m.ad_code , ret);
|
||||
def standardCity = RegionData.formatCity(cityMap.name)
|
||||
if(standardCity){
|
||||
saveData(standardCity, m.name, m.ad_code , ret);
|
||||
}else{
|
||||
println cityMap.name +" 没有标准化"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+11
-13
@@ -6,6 +6,7 @@ import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -47,14 +48,13 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-06-01"
|
||||
static def statDate = "2021-07-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
@@ -308,10 +308,7 @@ class YouDaDataParser {
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
latch.await()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def saveEsfHouseToDB(def data){
|
||||
@@ -367,15 +364,16 @@ class YouDaDataParser {
|
||||
row.ydId = data[2].toInteger()
|
||||
|
||||
row.fjCity = city
|
||||
def map = matchDistrict(city, row.region, row.districtName)
|
||||
if(!map){
|
||||
map = matchDistrict(city, row.region, row.address)
|
||||
}
|
||||
// def map = matchDistrict(city, row.region, row.districtName)
|
||||
// if(!map){
|
||||
// map = matchDistrict(city, row.region, row.address)
|
||||
// }
|
||||
def map = SeaweedDataMatchProcess.doMatch(city, row.region, row.districtName, row.address);
|
||||
if(map){
|
||||
row.fjRegion = map.region
|
||||
row.fjName = map.name
|
||||
row.lat = map.lat
|
||||
row.lng = map.lng
|
||||
row.lat = map.lat.toDouble()
|
||||
row.lng = map.lng.toDouble()
|
||||
row.location = map.lat.toString()+","+map.lng.toString()
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(map.id.toString());
|
||||
|
||||
+204
-7
@@ -7,16 +7,22 @@ import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.AMQP
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AddDistrictFieldValue {
|
||||
@@ -30,18 +36,26 @@ class AddDistrictFieldValue {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
|
||||
//第三方均价数据匹配
|
||||
// for(def city : citys){
|
||||
// def rets = findAllDataByCity(city)
|
||||
// match(rets)
|
||||
// }
|
||||
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
def rMap = getMatchDistrict(city)
|
||||
createResultValue(dists, rMap)
|
||||
}
|
||||
//第三方均价数据填补
|
||||
// for(def city : citys){
|
||||
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
// def rMap = getMatchDistrict(city)
|
||||
// createResultValue(dists, rMap)
|
||||
// }
|
||||
|
||||
//
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
|
||||
// addFinishDate(dists)
|
||||
addFieldValueByHouse(dists)
|
||||
}
|
||||
}
|
||||
|
||||
def match(def rets){
|
||||
@@ -118,6 +132,35 @@ class AddDistrictFieldValue {
|
||||
return rMap;
|
||||
}
|
||||
|
||||
def addFinishDate(dists){
|
||||
def updateIndex = 0
|
||||
def total = dists.size()
|
||||
for(def d : dists){
|
||||
if(!d.finish_building_date){
|
||||
def udata = new BasicDBObject()
|
||||
def newValue = FiledStrand.processFinishBuildingDate(d.live_date)
|
||||
if(!newValue && d.open_date){
|
||||
def date = d.open_date
|
||||
newValue = date.getYear()+1900
|
||||
}
|
||||
if(!newValue && d.build_date){
|
||||
newValue = FiledStrand.processFinishBuildingDate(d.build_date)
|
||||
}
|
||||
if(newValue){
|
||||
udata.put("finish_building_date", newValue)
|
||||
}
|
||||
if(udata.size()>0){
|
||||
updateIndex++
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def createResultValue(dists, rMap){
|
||||
def updateIndex = 0
|
||||
def total = dists.size()
|
||||
@@ -260,9 +303,163 @@ class AddDistrictFieldValue {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lastValue;
|
||||
}
|
||||
|
||||
|
||||
def addFieldValueByHouse(def dists){
|
||||
def updateIndex = new AtomicLong()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def total = dists.size()
|
||||
final CountDownLatch latch = new CountDownLatch(total)
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(d.estate_type2!="别墅" && (!d.finish_building_date || !d.build_type)){
|
||||
def udata = new BasicDBObject()
|
||||
def retMap = findHouses(d.city, d.region, "sell", d.name, d.alias)
|
||||
if(retMap){
|
||||
println retMap
|
||||
if(!d.finish_building_date && retMap.containsKey("build_date")){
|
||||
def dateMap = retMap.get("build_date")
|
||||
if(dateMap){
|
||||
def value = dateMap.keySet()[0]
|
||||
if(value){
|
||||
udata.put("finish_building_date", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!d.build_type && retMap.containsKey("height")){
|
||||
def heightMap = retMap.get("height")
|
||||
if(heightMap){
|
||||
def heights = heightMap.keySet()
|
||||
if(heights){
|
||||
def max = heights.max()
|
||||
if(max<=6){
|
||||
udata.put("build_type", "多层")
|
||||
}else if(max>6 && max<=11){
|
||||
udata.put("build_type", "小高层")
|
||||
}else if(max>11 && max<30){
|
||||
udata.put("build_type", "高层")
|
||||
}else if(max>30){
|
||||
udata.put("build_type", "超高层")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(udata.size()>0){
|
||||
updateIndex.addAndGet(1)
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
println updateIndex.get() +"/${total} "+d.city +" "+d.name+" "+udata
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{
|
||||
println totalCount.get() +" "+d.city +" "+d.name
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def name, def alias){
|
||||
|
||||
// QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
|
||||
// must(QueryBuilders.termQuery("cat", cat)).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def restMap = [:]
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-*", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
if(seawater.build_date){
|
||||
def newValue = FiledStrand.processFinishBuildingDate(seawater.build_date)
|
||||
if(newValue){
|
||||
def valueMap = restMap.get("build_date")
|
||||
if(!valueMap){
|
||||
valueMap = [:]
|
||||
restMap.put("build_date", valueMap)
|
||||
}
|
||||
def count = valueMap.get(newValue)
|
||||
if(!count){
|
||||
valueMap.put(newValue, 1)
|
||||
}else{
|
||||
valueMap.put(newValue, count + 1)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(seawater.height){
|
||||
def hightMap = restMap.get("height")
|
||||
if(!hightMap){
|
||||
hightMap = [:]
|
||||
restMap.put("height", hightMap)
|
||||
}
|
||||
def count = hightMap.get(seawater.height)
|
||||
if(!count){
|
||||
count = 1
|
||||
hightMap.put(seawater.height, count)
|
||||
}else{
|
||||
hightMap.put(seawater.height, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(restMap){
|
||||
def dateMap = restMap.get("build_date")
|
||||
if(dateMap){
|
||||
dateMap = dateMap.sort {a, b -> b.value <=> a.value}
|
||||
restMap.put("build_date", dateMap)
|
||||
}
|
||||
}
|
||||
return restMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -24,7 +24,7 @@ class DataProcess {
|
||||
|
||||
static main(args) {
|
||||
//建行信用卡城市(临汾--晋城).xlsx
|
||||
def filePath = "D:\\数据\\小区\\地图\\张家口.xlsx"
|
||||
def filePath = "D:\\数据\\小区\\地图\\兰州.xlsx"
|
||||
def task = new DataProcess();
|
||||
// task.batchSaveDataToDB(filePath)
|
||||
task.mapBatchSaveDataToDB(filePath)
|
||||
@@ -121,7 +121,7 @@ class DataProcess {
|
||||
if(datas){
|
||||
def keys = []
|
||||
for(def data : datas){//20,21,25,
|
||||
if(data['I'] && [20,21,25,26,27].contains(data['I'].toString().toDouble().toInteger())){
|
||||
if(data['I'] && [20,21,26,27].contains(data['I'].toString().toDouble().toInteger())){
|
||||
def key = data['A']+"##"+data['B']+"##"+data['H']
|
||||
if(!keys.contains(key) && data['H'] && data['H'].size()>2){
|
||||
keys << key
|
||||
|
||||
+42
-5
@@ -5,15 +5,22 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/***
|
||||
* 地图小区处理
|
||||
*/
|
||||
class MapDistrictProcess {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
|
||||
static main(args) {
|
||||
def task = new MapDistrictProcess();
|
||||
task.run()
|
||||
@@ -28,17 +35,18 @@ class MapDistrictProcess {
|
||||
def run(){
|
||||
|
||||
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
|
||||
def files = ["青岛.xlsx","沈阳.xlsx","哈尔滨.xlsx","张家口.xlsx","大连.xlsx"]
|
||||
//"乌鲁木齐.xlsx","烟台.xlsx","保定.xlsx","贵阳.xlsx","湖州.xlsx","惠州.xlsx","吉林.xlsx","遵义.xlsx","宜昌.xlsx","芜湖.xlsx","兰州.xlsx"
|
||||
def files = ["兰州.xlsx"]
|
||||
for(def fileName : files){
|
||||
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
|
||||
typeFilter(filePath)
|
||||
nameStandard(filePath);
|
||||
dataFilter(filePath)
|
||||
houseCalculate(filePath);
|
||||
////// //统计是否存在多个区域
|
||||
//统计是否存在多个区域
|
||||
matchDistDatas(filePath)
|
||||
moreRegionCalculate(filePath)
|
||||
|
||||
//
|
||||
addFlag(filePath)
|
||||
mapMatch(filePath);
|
||||
baiduTypeFilter(filePath);
|
||||
@@ -50,7 +58,8 @@ class MapDistrictProcess {
|
||||
addFeature(filePath)
|
||||
houseMatch(filePath);
|
||||
|
||||
// nameScore(filePath);
|
||||
nameScore(filePath);
|
||||
thirdPartyData(filePath)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -212,7 +221,7 @@ class MapDistrictProcess {
|
||||
|
||||
def addFlag(def filePath){
|
||||
def index = 0
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
index++
|
||||
if(!row['I']) {
|
||||
//根据别名查找
|
||||
@@ -438,6 +447,34 @@ class MapDistrictProcess {
|
||||
},0)
|
||||
|
||||
}
|
||||
def thirdPartyData(filePath){
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(row['I'] && [20,21,25,26,27].contains(row['I'].toString().toDouble().toInteger())){
|
||||
def rets = findIsExistsOtherWebsite(row['A'], row['B'], row['H'])
|
||||
if(rets){
|
||||
row['W'] = rets.join(",")
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def findIsExistsOtherWebsite(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("region",region)
|
||||
q.put("name", name)
|
||||
def dbcursor = districtDetailColl.find(q)
|
||||
def rets = []
|
||||
for(def r : dbcursor){
|
||||
if(!rets.contains(r.root)){
|
||||
rets << r.root
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getRegionByLocation(lng, lat){
|
||||
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
|
||||
|
||||
+3
@@ -25,6 +25,9 @@ class SeaweedDataMatchProcess {
|
||||
rMap.city = obj.city
|
||||
rMap.region = obj.region
|
||||
rMap.name = obj.name
|
||||
rMap.lat = obj.lat/10000000000.0
|
||||
rMap.lng = obj.lng/10000000000.0
|
||||
rMap.id = obj._id.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
package com.fangjia.script.gj.districtwash
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
|
||||
class AddDetailDistrict {
|
||||
|
||||
def detailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDetailDistrict();
|
||||
// task.run();
|
||||
// task.delRepeatDist
|
||||
task.addEstateType()
|
||||
}
|
||||
|
||||
def addEstateType(){
|
||||
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]
|
||||
for(def city : citys){
|
||||
def keys = []
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","estate_type2"])
|
||||
for(def d : districts) {
|
||||
if(!d.estate_type2){
|
||||
def estateType2 = "普通住宅"
|
||||
if(d.name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(d.name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
DBObject update = new BasicDBObject();
|
||||
update.put('$set', new BasicDBObject("estate_type2", estateType2).append("m_date", new Date()));
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
|
||||
}
|
||||
}
|
||||
println city
|
||||
}
|
||||
}
|
||||
|
||||
def delRepeatDist(){
|
||||
def citys = CcbCommonUtils.citys
|
||||
for(def city : citys){
|
||||
def keys = []
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index++
|
||||
def key = d.city +"##"+d.region+"##"+d.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}else{
|
||||
println index +" "+d.region+" "+d.name
|
||||
DBObject update = new BasicDBObject();
|
||||
update.put('$set', new BasicDBObject("status", 1).append("m_date", new Date()));
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = ["沈阳", "哈尔滨", "吉林", "遵义", "淄博", "西宁", "青岛", "大连", "宁波"]//CcbCommonUtils.citys;
|
||||
//第三方均价数据匹配
|
||||
for (def city : citys) {
|
||||
def rets = findAllDataByCity(city)
|
||||
def rMap = getMatchDistrict(city)
|
||||
toDB(rets, rMap)
|
||||
}
|
||||
}
|
||||
|
||||
def toDB(def rets, def rMap){
|
||||
def saveDistKeys = []
|
||||
def total = rets.size()
|
||||
def index = 0
|
||||
def addIndex = 0
|
||||
for(def d : rets){
|
||||
index++
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
def name = NameProcess.nameStandard(d.city, newRegion, d.name)
|
||||
if(!name){
|
||||
continue
|
||||
}
|
||||
def flag = NameProcess.filterName(name)
|
||||
def key = null
|
||||
if(newRegion){
|
||||
key = d.city+"#"+newRegion+"#"+name
|
||||
}
|
||||
if(flag && !saveDistKeys.contains(key)){
|
||||
def gdData = FangjiaClientUtils.getMapData(d.city, newRegion, name, d.address,"", "高德")
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(gdData){
|
||||
|
||||
if(!newRegion){
|
||||
newRegion = RegionData.formatRegion(d.city, gdData.district)
|
||||
if(newRegion){
|
||||
key = d.city+"#"+newRegion+"#"+name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(key && gdData && gdData.lng){
|
||||
def matchMap = SeaweedDataMatchProcess.doMatch(d.city, newRegion, name, d.address);
|
||||
if(!matchMap){
|
||||
addIndex++
|
||||
if(gdData.lng){
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(gdData.lng.toDouble(), gdData.lat.toDouble())
|
||||
lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
|
||||
lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
|
||||
lng = (gdData.lng.toDouble()* 10000000000).longValue()
|
||||
lat = (gdData.lat.toDouble()* 10000000000).longValue()
|
||||
}
|
||||
def newAddress = DataUtil.formateAddress(d.city, d.address)
|
||||
def udata = createFieldValue(d, rMap)
|
||||
udata.put("lng", lng)
|
||||
udata.put("lat", lat)
|
||||
println index+"/${addIndex}/${total} "+d.city+" "+newRegion+" "+name+" "+d.root+" "+udata
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", name)
|
||||
udata.put("address", newAddress)
|
||||
|
||||
udata.put("lng2", lng2)
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("status", 0)
|
||||
udata.put("cat", "district")
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.save(udata)
|
||||
saveDistKeys << key
|
||||
}
|
||||
}
|
||||
// println d.city+" "+ d.region+" "+d.name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def createFieldValue(d, rMap){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
def udata = new BasicDBObject()
|
||||
if(sourceMap){
|
||||
if(!d.address){
|
||||
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("address", lastValue)
|
||||
}
|
||||
}
|
||||
if(!d.all_area){
|
||||
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("all_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.build_area){
|
||||
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("build_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.virescence_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("virescence_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.cubage_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("cubage_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.finish_building_date){
|
||||
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("finish_building_date", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.household_count){
|
||||
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("household_count", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.car_site_count){
|
||||
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("car_site_count", lastValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
return udata
|
||||
}
|
||||
|
||||
def getValuesByField(sourceMap, field, estateType){
|
||||
def map = [:]
|
||||
sourceMap.each{k,v->
|
||||
def oldValue = v.get(field);
|
||||
if(oldValue){
|
||||
def newValue = null;
|
||||
if(["address"].contains(field)){
|
||||
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
|
||||
}
|
||||
if(["all_area","build_area"].contains(field)){
|
||||
newValue = FiledStrand.processArea(oldValue)
|
||||
}
|
||||
if(["build_date"].contains(field)){
|
||||
newValue = FiledStrand.processFinishBuildingDate(oldValue)
|
||||
}
|
||||
if(["house_count"].contains(field)){
|
||||
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
|
||||
}
|
||||
if(["greening_rate"].contains(field)){
|
||||
newValue = FiledStrand.processVirescencePercent(oldValue)
|
||||
}
|
||||
if(["volumetric_rate"].contains(field)){
|
||||
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
|
||||
}
|
||||
if(["car_site_desc","car_site"].contains(field)){
|
||||
newValue = FiledStrand.processCarSite(oldValue)
|
||||
}
|
||||
if(newValue){
|
||||
def count = map.get(newValue)
|
||||
if(!count){
|
||||
map.put(newValue, 1)
|
||||
}else{
|
||||
map.put(newValue, count+1)
|
||||
}
|
||||
|
||||
v.put(field, newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def lastValue = null
|
||||
if(map && map.size()>0){
|
||||
map = map.sort {a, b -> b.value <=> a.value}
|
||||
def values = map.values()
|
||||
if(values.size()==1){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def count1 = map.values()[0]
|
||||
def count2 = map.values()[1]
|
||||
if(count1 > count2){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def rets = [map.keySet()[0],map.keySet()[1]]
|
||||
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
|
||||
for(def root : roots){
|
||||
def data = sourceMap.get(root)
|
||||
if(data){
|
||||
def value = data.get(field)
|
||||
if(rets.contains(value)){
|
||||
lastValue = value
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValue;
|
||||
}
|
||||
|
||||
|
||||
def getMatchDistrict(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("match_name", new BasicDBObject('$exists', true))
|
||||
def dbcursor = detailColl.find(q)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
if(!sourceMap){
|
||||
sourceMap = [:]
|
||||
rMap.put(key, sourceMap)
|
||||
}
|
||||
def value = sourceMap.get(d.root)
|
||||
if(!value){
|
||||
sourceMap.put(d.root, d)
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def findAllDataByCity(def city){
|
||||
def q = new BasicDBObject("city", city);
|
||||
q.put("match_name", new BasicDBObject('$exists', false))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
|
||||
def dbcursor = detailColl.find(q).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-1
@@ -1,8 +1,9 @@
|
||||
package com.fangjia.script.gj
|
||||
package com.fangjia.script.gj.districtwash
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.datastore.FieldValidatorUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.Utils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
class Point {
|
||||
Double lat;
|
||||
Double lng;
|
||||
|
||||
public Point(Double lng, Double lat){
|
||||
this.lat = lat
|
||||
this.lng = lng
|
||||
}
|
||||
|
||||
Double getLat() {
|
||||
return lat
|
||||
}
|
||||
|
||||
void setLat(Double lat) {
|
||||
this.lat = lat
|
||||
}
|
||||
|
||||
Double getLng() {
|
||||
return lng
|
||||
}
|
||||
|
||||
void setLng(Double lng) {
|
||||
this.lng = lng
|
||||
}
|
||||
}
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.awt.geom.Point2D
|
||||
|
||||
class PointUtil {
|
||||
|
||||
public static final DBCollection BUSINESS_RANGE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border", "mongo.db.fangjia.com")
|
||||
|
||||
/**
|
||||
* 判断当前位置是否在多边形区域内
|
||||
*
|
||||
* @param checkPoint 当前经纬度
|
||||
* @param areaAround 围栏范围
|
||||
* @return
|
||||
*/
|
||||
public static boolean checkIsInPolygon(Point checkPoint, List<Point> areaAround) {
|
||||
Point2D.Double point = new Point2D.Double(checkPoint.getLat(), checkPoint.getLng());
|
||||
List<Point2D.Double> pointList = new ArrayList<>();
|
||||
for (Point everyPoint : areaAround) {
|
||||
Point2D.Double polygonPoint = new Point2D.Double(everyPoint.getLat(), everyPoint.getLng());
|
||||
pointList.add(polygonPoint);
|
||||
}
|
||||
return checkIsPtInPoly(point, pointList);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断点是否在多边形内,如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
|
||||
*
|
||||
* @param point 检测点
|
||||
* @param pts 多边形的顶点
|
||||
* @return 点在多边形内返回true, 否则返回false
|
||||
*/
|
||||
public static boolean checkIsPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
|
||||
int N = pts.size();
|
||||
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
|
||||
boolean boundOrVertex = true;
|
||||
//cross points count of x
|
||||
int intersectCount = 0;
|
||||
//浮点类型计算时候与0比较时候的容差
|
||||
double precision = 2e-10;
|
||||
//neighbour bound vertices
|
||||
Point2D.Double p1, p2;
|
||||
//当前点
|
||||
Point2D.Double p = point;
|
||||
//left vertex
|
||||
p1 = pts.get(0);
|
||||
//check all rays
|
||||
for (int i = 1; i <= N; ++i) {
|
||||
if (p.equals(p1)) {
|
||||
//p is an vertex
|
||||
return boundOrVertex;
|
||||
}
|
||||
//right vertex
|
||||
p2 = pts.get(i % N);
|
||||
//ray is outside of our interests
|
||||
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
|
||||
p1 = p2;
|
||||
//next ray left point
|
||||
continue;
|
||||
}
|
||||
//ray is crossing over by the algorithm (common part of)
|
||||
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
|
||||
//x is before of ray
|
||||
if (p.y <= Math.max(p1.y, p2.y)) {
|
||||
//overlies on a horizontal ray
|
||||
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
|
||||
return boundOrVertex;
|
||||
}
|
||||
//ray is vertical
|
||||
if (p1.y == p2.y) {
|
||||
//overlies on a vertical ray
|
||||
if (p1.y == p.y) {
|
||||
return boundOrVertex;
|
||||
//before ray
|
||||
} else {
|
||||
++intersectCount;
|
||||
}
|
||||
} else {
|
||||
//cross point on the left side
|
||||
//cross point of y
|
||||
double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
|
||||
//overlies on a ray
|
||||
if (Math.abs(p.y - xinters) < precision) {
|
||||
return boundOrVertex;
|
||||
}
|
||||
//before ray
|
||||
if (p.y < xinters) {
|
||||
++intersectCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//special case when ray is crossing through the vertex
|
||||
//p crossing over p2
|
||||
if (p.x == p2.x && p.y <= p2.y) {
|
||||
//next vertex
|
||||
Point2D.Double p3 = pts.get((i + 1) % N);
|
||||
//p.x lies between p1.x & p3.x
|
||||
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
|
||||
++intersectCount;
|
||||
} else {
|
||||
intersectCount += 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
//next ray left point
|
||||
p1 = p2;
|
||||
}
|
||||
//偶数在多边形外
|
||||
if (intersectCount % 2 == 0) {
|
||||
return false;
|
||||
} else {
|
||||
//奇数在多边形内
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(def a){
|
||||
new PointUtil().run()
|
||||
}
|
||||
def run(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", "淞南")
|
||||
def obj = BUSINESS_RANGE_COLL.findOne(q)
|
||||
if(obj){
|
||||
def pointStr = obj.border
|
||||
if(pointStr){
|
||||
def pointArrs = pointStr.split(";")
|
||||
def list = []
|
||||
for(def s : pointArrs){
|
||||
def arr = s.split(",")
|
||||
list << new Point(arr[0].toDouble(), arr[1].toDouble())
|
||||
}
|
||||
|
||||
println checkIsInPolygon(new Point(121.491478,31.327947), list)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-6
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","福州")
|
||||
// q.put("source", "安居客")
|
||||
q.put("city","福州")
|
||||
q.put("source", "城市房产")
|
||||
q.put("s_date",202012)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = "anjuke.com/"
|
||||
def prefix = ".cityhouse.cn/ha/"
|
||||
def keyList =new HashSet()
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
@@ -77,11 +77,11 @@ class CrawlDistrictIdSelect {
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com"))
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
@@ -92,7 +92,7 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "anjuke.com"
|
||||
map.root = "cityhouse.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
|
||||
@@ -1,7 +1,35 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
Date date = new Date(1617211269868);
|
||||
print dateFormat.format(date);
|
||||
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
//Date date = new Date(1617211269868);
|
||||
//print dateFormat.format(date);
|
||||
DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
|
||||
def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
|
||||
ExcelUtils.modify(path,{d->
|
||||
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
|
||||
d['G'] = districts.size()
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['B'])
|
||||
q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
|
||||
def obj = CITY_INFO_CONN.findOne(q);
|
||||
if(obj){
|
||||
d['H'] = obj.price_coverage*100
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+5
-2
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.thirdPriceCrawl
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.solr.common.util.Hash
|
||||
|
||||
@@ -13,8 +14,9 @@ class SynchDistrictPrice {
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
"anjuke.com":"安居客"]
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["zhuge.com":"诸葛找房"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -39,6 +41,7 @@ class SynchDistrictPrice {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("root", root)
|
||||
q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")));
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
|
||||
@@ -921,13 +921,13 @@ class DateUtils {
|
||||
|
||||
public static String getRandomDate(def startDate, def endDate){
|
||||
Date randomDate = randomDate(startDate, endDate);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
|
||||
String result = format.format(randomDate);
|
||||
return result;
|
||||
}
|
||||
|
||||
static main(args){
|
||||
|
||||
println getRandomDate("2021-08-01","2021-08-31")
|
||||
// nowFirstDay();
|
||||
// nowLastDay();
|
||||
/*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
|
||||
@@ -955,8 +955,9 @@ class DateUtils {
|
||||
// Date randomDate = randomDate("2010-09-20", "2011-09-22");
|
||||
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
|
||||
// String result = format.format(randomDate);
|
||||
System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
|
||||
// System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
|
||||
|
||||
println DateUtils.getDayOfWeek(new Date())
|
||||
|
||||
}
|
||||
|
||||
|
||||
+4
-4
@@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
|
||||
def city = "À³Îß"
|
||||
def destCity = "¼ÃÄÏ"
|
||||
def region = ""
|
||||
def destRegion = "À³Îß"
|
||||
def city = "ÂåÑô"
|
||||
def destCity = "ÂåÑô"
|
||||
def region = "¼ªÀû"
|
||||
def destRegion = "ÃϽò"
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user