This commit is contained in:
2021-08-03 20:06:38 +08:00
parent be891b3229
commit 9017dd4c2a
20 changed files with 931 additions and 228 deletions
@@ -1,6 +1,7 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -40,12 +41,11 @@ class BigdataCommunityBatch extends BaseBatch{
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rows.addAll(rets)
println city +" "+rets.size()
@@ -57,10 +57,10 @@ class BigdataCommunityBatch extends BaseBatch{
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false);
println "txt 生成"
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -70,7 +70,7 @@ class BigdataCommunityBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, distCodeMap, dist)
def ret = createData(cityCodeDataMap, distCodeMap, dist)
if(ret){
rets << ret
}
@@ -87,10 +87,10 @@ class BigdataCommunityBatch extends BaseBatch{
return rets;
}
def createData(def cityCodeDatas, def distCodeMap, def dist){
def createData(def cityCodeDataMap, def distCodeMap, def dist){
def ret = []
try{
ret << cityCodeDatas.city_code
ret << cityCodeDataMap.get(dist.city)
if(dist.region){
ret << distCodeMap.get(dist.region)
}else{
@@ -101,9 +101,17 @@ class BigdataCommunityBatch extends BaseBatch{
ret << dist.region
ret << dist.name
ret << dist.address
if(dist.lat){
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
if(dist.lat2){
def lat = dist.lat2/10000000000d
def lng = dist.lng2/10000000000d
def loactionMap = GeoUtils.bd09ToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
@@ -120,7 +128,7 @@ class BigdataCommunityBatch extends BaseBatch{
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "物业小区"
ret << "pa100"
ret << ""
if(dist.address){
if(dist.address.toString().endsWith("弄")){
@@ -27,8 +27,8 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -44,35 +44,43 @@ class BigdataDealBatch extends BaseBatch{
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def trades = getCityData(city)
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other")
// rows.addAll(rets)
def trades = getCityData(city)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
def youDataTrade = getYouDaData()
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" 成交共:"+rets.size()
if(city=="上海"){
def youDataTrade = getYouDaData()
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
println city +" "+youdaDatas.size()
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rets)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
cityDatas.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false);
def fileName = "/home/goojia/ccb/bigdata_deal.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
def rets = []
rets << headers
rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -114,9 +122,9 @@ class BigdataDealBatch extends BaseBatch{
if(!obj){
return ret;
}
ret << cityCodeDatas.city_code
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
ret << trade.city
if(trade.fjRegion){
ret << distCodeMap.get(trade.fjRegion)
}else{
@@ -215,11 +223,11 @@ class BigdataDealBatch extends BaseBatch{
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << ""
ret << ""
ret << ""
@@ -240,7 +248,7 @@ class BigdataDealBatch extends BaseBatch{
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
@@ -248,9 +256,9 @@ class BigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
ret << cityCodeDatas.city_code
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
ret << trade.city
if(trade.region){
ret << distCodeMap.get(trade.region)
}else{
@@ -317,7 +325,7 @@ class BigdataDealBatch extends BaseBatch{
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
ret << ""
ret << ""
ret << ""
@@ -373,15 +381,23 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = null;
if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
}else{
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
}
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date()))
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate)
if(trades){
tradesList.addAll(trades)
}
@@ -12,6 +12,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -54,7 +55,8 @@ class BigdataForsaleBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
@@ -63,15 +65,19 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def index = 0
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
index++
def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, distIds,20200701, 20210731);
}
}
def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap,districts, distIds,startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
@@ -85,36 +91,42 @@ class BigdataForsaleBatch extends BaseBatch{
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret){
def cityCode = cityCodeDatas.city_code
def regionCode = distCodeMap.get(rMap.region)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = rMap.region
m.region_code = regionCode
m.district_id = rMap.get("_id").toString()
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
def commId = rMap.get("_id").toString()
if(distIds.contains(commId)){
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret && ret.size()<100){
ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
}
if(ret){
def cityCode = cityCodeDataMap.get(city)
def regionCode = distCodeMap.get(rMap.region)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = rMap.region
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
}
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -124,14 +136,27 @@ class BigdataForsaleBatch extends BaseBatch{
})
}
latch.await()
def fileName = "/home/goojia/ccb/bigdata_forsale.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt"
}
println city + " 房源总量:"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true);
def headers = []
if(index == 1){
headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
}
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
// def rets = []
// rets << headers
// rets.addAll(allDatas)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据")
}
@@ -236,6 +261,15 @@ class BigdataForsaleBatch extends BaseBatch{
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
@@ -358,8 +392,9 @@ class BigdataForsaleBatch extends BaseBatch{
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
@@ -518,4 +553,14 @@ class BigdataForsaleBatch extends BaseBatch{
println ids.size()
}
def getDistrictIds(def cityid){
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
def distIds = new HashSet()
sqlInstance.eachRow(sql,{it->
distIds << it.commid
});
return distIds;
}
}
@@ -7,7 +7,7 @@ import com.mongodb.DBCollection
class CcbCommonUtils {
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
public static def citys = ["成都","沈阳","大连","北京", "上海","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","重庆","南京","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -23,21 +23,10 @@ class CcbCommonUtils {
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def findCityCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def count = dbcursor.count()
if(count >1){
def data = null
for(def db : dbcursor){
if(db.city_code){
data = db
break
}
}
return data;
}
public static def findCityCodeValue(){
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
return cityCodeMap;
}
public static def findAllDistrictCodeValue(city){
@@ -105,4 +94,45 @@ class CcbCommonUtils {
return lists
}
public static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
}
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
results << headers
results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8");
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\staging_district.txt",headers, "utf-8", false);
}
@@ -76,9 +76,9 @@ class StagingDistrictBatch extends BaseBatch{
def getFenQiMap(def dists){
def fenQiMap = [:]
for(def d : dists){
if(d.master_district_name){
// if(d.master_district_name){
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// }
}
return fenQiMap;
}
@@ -19,7 +19,7 @@ import com.mongodb.DBCollection
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -30,7 +30,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]//CcbCommonUtils.citys;
def rows = []
def index = 0
for(def city : citys){
@@ -166,7 +166,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
def dbcursor = AVERAGE_STATISTICS.find(q)
def map = [:]
for(def db : dbcursor){
map.put(db.region+"##"+db.name, db)
if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
map.put(db.region+"##"+db.name, db)
}
}
return map
}
@@ -698,6 +698,28 @@ class CommonDatautils {
return rets;
}
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", new BasicDBObject('$in', regions))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllDistrictByQuery(def q, def cat, def fields){
@@ -2,6 +2,8 @@ package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
@@ -63,7 +65,7 @@ class AddThirdPriceDistrict {
def resAddrMap = [:]
def filterEstateTypeMap = [:]
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -82,14 +84,77 @@ class AddThirdPriceDistrict {
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
}
def run(){
exportDatas();
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// addInfo();
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
// exportDatas();
// getDataType("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx")
addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
}
def dataFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
def name = row['C']
if(name){
def flag = NameProcess.filterName(name)
if(!flag){
row['T'] = 1
}else{
if(name.toString().length()<3){
if(row['N'] && row['N']>=10 && !row['J']){
row['I'] = 25
}else{
row['I'] = 18
}
}else{
def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){
row['I'] = 19
}
}
}
}else{
if(!name){
row['I'] = 1
}
}
});
println "3.名称过滤完成"
}
def addNewMapInfo(def path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['I']){
def gdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "高德")
if(gdData){
row['F'] = gdData.district
row['G'] = gdData.name
row['H'] = gdData.address
row['I'] = gdData.type
row['J'] = gdData.score
row['K'] = gdData.lng
row['L'] = gdData.lat
}
def bdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "百度")
if(bdData){
row['M'] = bdData.district
row['N'] = bdData.name
row['O'] = bdData.address
row['P'] = bdData.type
row['Q'] = bdData.score
row['R'] = bdData.lng
row['S'] = bdData.lat
}
}
},0);
}
def addCrawlMapInfo( def path){
ExcelUtils.multiThreadingModify(path, { row ->
def flag = row['R']
@@ -105,27 +170,27 @@ class AddThirdPriceDistrict {
row['K'] = data.lat
}
def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
def bdRegion =null
def bdName = null
def bdAddress = null
def bdLng = null
def bdLat = null
def bdEstateType = null
if(baiduRets){
bdRegion = baiduRets.region
bdName = baiduRets.name
bdAddress = baiduRets.address
bdLng = baiduRets.lng
bdLat = baiduRets.lat
bdEstateType = baiduRets.estateType
}
row['L'] = bdRegion
row['M'] = bdName
row['N'] = bdAddress
row['O'] = bdEstateType
row['P'] = bdLng
row['Q'] = bdLat
// def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
// def bdRegion =null
// def bdName = null
// def bdAddress = null
// def bdLng = null
// def bdLat = null
// def bdEstateType = null
// if(baiduRets){
// bdRegion = baiduRets.region
// bdName = baiduRets.name
// bdAddress = baiduRets.address
// bdLng = baiduRets.lng
// bdLat = baiduRets.lat
// bdEstateType = baiduRets.estateType
// }
// row['L'] = bdRegion
// row['M'] = bdName
// row['N'] = bdAddress
// row['O'] = bdEstateType
// row['P'] = bdLng
// row['Q'] = bdLat
}
},0);
@@ -502,8 +567,8 @@ class AddThirdPriceDistrict {
return thirdPartyColl.findOne(q)
}
def addInfo(){
def addInfo(filePath){
province = "江西"
// def city = "苏州"
// def datas = findMatchErrorDatas(city);
// println "共【"+datas.size()+"】条"
@@ -512,7 +577,7 @@ class AddThirdPriceDistrict {
// def resMap = groupByName(datas);
def index = 0;
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx", { row ->
ExcelUtils.multiThreadingModify(filePath, { row ->
def flag = row['R']
def dgDatas = row['G']
// if(!flag){
@@ -956,11 +1021,11 @@ class AddThirdPriceDistrict {
def queryPoiMap(def city, def region, def name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and adname = '" + region + "'")
sqlBuffer.append(" and name = '" + name + "'")
sqlBuffer.append(" and source = '高德'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and source = '高德' order by score() desc limit 1")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def map = [:]
while (rs.next()) {
@@ -969,8 +1034,11 @@ class AddThirdPriceDistrict {
map.name = rs.getString("name")
map.address = rs.getString("address")
map.type = rs.getString("type")
map.lng = rs.getString("location.lon")
map.lat = rs.getString("location.lat")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
}
return map;
}
@@ -64,24 +64,29 @@ class EsSearchUtils {
sqlBuffer.append(" and region = '"+region+"'")
sqlBuffer.append(" and district_name = '"+name+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" limit 100")
sqlBuffer.append(" limit 200")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def rets = []
while (rs.next()){
def height = rs.getInt("height")
def direction = rs.getString("direction")
def fitment = rs.getString("fitment")
def room = rs.getInt("room")
def hall = rs.getInt("hall")
def toilet = rs.getInt("toilet")
def map = [:]
map.height = height
map.direction = direction
map.fitment = fitment
map.room = room
map.hall = hall
map.toilet = toilet
rets << map
if(room){
def height = rs.getInt("height")
def direction = rs.getString("direction")
def fitment = rs.getString("fitment")
def hall = rs.getInt("hall")
def toilet = rs.getInt("toilet")
def area = rs.getInt("area")
def map = [:]
map.height = height
map.direction = direction
map.fitment = fitment
map.room = room
map.hall = hall
map.toilet = toilet
map.area = area
rets << map
}
}
return rets;
}
@@ -50,14 +50,14 @@ class GaoDeMapCrawl extends BaseBatch{
// def categorys = ["141202", "141203"]//中学,小学
// def prefix = "district_"
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
def prefix = "district_"
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def prefix = "road_"
// def categorys = ["190301"]//道路相关
def prefix = "village_"
def categorys = ["190108","190109"]//村庄
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
// def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
@@ -119,7 +119,7 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys =["无锡"]
def citys =["南昌"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
@@ -28,8 +28,8 @@ class RegionPointCrawl {
static main(args) {
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
def provinces = ["江西省"]
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
@@ -72,7 +72,7 @@ class RegionPointCrawl {
for(def cityMap : cityList){
if(cityMap.level == "city"){
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
saveGaoDeRegionCode(province, cityMap.districts);
saveGaoDeRegionCode(province, cityMap.name, cityMap.districts);
}
}
}
@@ -81,10 +81,10 @@ class RegionPointCrawl {
}
}
def saveGaoDeRegionCode(province, districts){
def saveGaoDeRegionCode(province, cityName,districts){
if(districts){
for(def dist : districts){
if(dist.level=="district"){
if(dist.level=="district" || cityName=="东莞市"){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
@@ -221,4 +221,3 @@ class RegionPointCrawl {
return KeyList.get(count);
}
}
H
@@ -12,8 +12,10 @@ class FileUtils {
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
if(headers){
sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
@@ -15,7 +15,19 @@ class BestDistrictPriceExport {
static main(args) {
def task = new BestDistrictPriceExport();
task.run()
//task.run()
task.run2()
}
def run2(){
def filePath = "d:\\数据\\广发\\楼盘.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
def stat = findDistrictPrice(row['A'], row['B'], row['D'], "20210801")
if(stat){
row['S'] = stat.avg_price
}
println row['A']+" "+row['B']+" "+ row['C']+" "+row['S']
})
}
def run(){
@@ -24,10 +24,10 @@ class DataProcess {
static main(args) {
//建行信用卡城市(临汾--晋城).xlsx
def filePath = "D:\\数据\\建行银行\\建行信用卡入库小区V1.xlsx"
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx"
def task = new DataProcess();
// task.batchSaveDataToDB(filePath)
// task.mapBatchSaveDataToDB(filePath)
task.mapBatchSaveDataToDB(filePath)
// task.calcutePrice(filePath);
// task.calcutePrice2();
// task.deleteRepDist("株洲");
@@ -35,7 +35,7 @@ class DataProcess {
// task.standartName(filePath);
task.saveGaoDeMap(filePath);
// task.saveGaoDeMap(filePath);
// task.updateDBAddress();
// task.updateRegion()
@@ -120,13 +120,10 @@ class DataProcess {
def lastDatas = []
if(datas){
def keys = []
for(def data : datas){
if(data['A']=="厦门" && data['I'] && data['I'].toString().startsWith("24")){
for(def data : datas){//20,21,25,
if(data['I'] && [26,27].contains(data['I'].toString().toDouble().toInteger())){
def key = data['A']+"##"+data['B']+"##"+data['H']
if(!keys.contains(key) && data['H'].size()>2){
// if(data['I'].toInteger()==22){//&& data['N']<3
// continue;
// }
if(!keys.contains(key) && data['H'] && data['H'].size()>2){
keys << key
lastDatas << data
}
@@ -168,7 +165,7 @@ class DataProcess {
addIndex++
def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null)
println "${addIndex}【新增入库】"+ values
// FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
}else{
println "【已存在】 ${d['A']} ${d['B']} ${d['H']}"
}
@@ -5,6 +5,7 @@ import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
@@ -26,8 +27,9 @@ class ExportData {
static main(args) {
def task = new ExportData();
// task.exportThird()
task.exportMap();
task.exportThird()
// task.exportMap();
}
def exportMap(){
@@ -72,7 +74,7 @@ class ExportData {
def exportThird(){
def citys = ["ÖØÇì"]
def citys = ["Äϲı"]
for(def city : citys){
def rets = []
rets << ["城市","区域","名称","别名"]
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
@@ -7,6 +8,7 @@ import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import java.util.regex.Pattern
@@ -15,32 +17,39 @@ class MapDistrictProcess {
static main(args) {
def task = new MapDistrictProcess();
task.run()
// task.updateRegion()
}
/**
* 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路
* 20:不同区域新增(库中无) 21:分期新增 22:分数在0.6以上
* 20:不同区域新增(库中无) 21:分期新增 23:不同区域,距离相近 25:无匹配,可新增 22:有匹配,距离1000米以下,相似度在0.6以上 24:疑似匹配有误 27:人工判定入库
* @return
*/
def run(){
def files = ["武汉.xlsx"]
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
def files = ["东莞.xlsx"]
for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
// typeFilter(filePath)
// nameStandard(filePath);
// dataFilter(filePath)
matchDistDatas(filePath)
typeFilter(filePath)
nameStandard(filePath);
dataFilter(filePath)
houseCalculate(filePath);
////// //统计是否存在多个区域
matchDistDatas(filePath)
moreRegionCalculate(filePath)
addFlag(filePath)
mapMatch(filePath);
baiduTypeFilter(filePath);
baiduMap(filePath);
// baiduMap(filePath);
distanceFilter(filePath)
predictType(filePath)
noMatchFlag(filePath)
matchFlag(filePath)
addFeature(filePath)
houseMatch(filePath);
predictType(filePath)
// nameScore(filePath);
}
@@ -55,7 +64,12 @@ class MapDistrictProcess {
row['I'] = 1
}else{
if(name.toString().length()<3){
row['I'] = 18
if(row['N'] && row['N']>=10 && !row['J']){
row['I'] = 25
}else{
row['I'] = 18
}
}else{
def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){
@@ -70,7 +84,7 @@ class MapDistrictProcess {
}
});
println "2.名称过滤完成"
println "3.名称过滤完成"
}
def typeFilter(filePath){
@@ -84,15 +98,30 @@ class MapDistrictProcess {
println "1.类型过滤完成"
}
def baiduTypeFilter(filePath){
def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"]
ExcelUtils.multiThreadingModify(filePath, { row ->
def type = row['S']
if(type && !types.contains(type)){
if(type && row['Q'] && row['Q']==row['H'] && !types.contains(type)){
row['I'] = 1
}
});
println "1.类型过滤完成"
println "9.百度类型过滤完成"
}
def distanceFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']){
if(row['X'] && row['Y']){
if(row['X']<=500 && row['Y']>=0.5){
row['I'] = 12
}
}
}
});
println "10.距离过滤完成"
}
def nameStandard(filePath){
@@ -105,12 +134,12 @@ class MapDistrictProcess {
}
}, 0);
println "1.名称标准化完成"
println "2.名称标准化完成"
}
def matchDistDatas(def path){
def index = 0
ExcelUtils.modify(path,{ List row ->
ExcelUtils.multiThreadingModify(path,{ List row ->
index ++
if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
@@ -126,13 +155,16 @@ class MapDistrictProcess {
def maplat = baidu.y
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
if(score<0.3){
score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['M'])
}
row['Y'] = score
}
}
println index +" "+row
}
},0)
println "3.小区匹配完成"
println "5.小区匹配完成"
}
def houseCalculate(filePath){
@@ -141,6 +173,7 @@ class MapDistrictProcess {
row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H'])
}
});
println "4.房源统计完成"
}
def moreRegionCalculate(filePath){
@@ -161,8 +194,12 @@ class MapDistrictProcess {
}
}
if(!regions.contains(row['B'])){
row['O'] = regions.toString()
row['I'] = 20
if(row['X'] && row['X']<=500 && row['Y'] && row['Y']>=0.5){
row['I'] = 23
}else{
row['O'] = regions.toString()
row['I'] = 20
}
}else if(regions.contains(row['B'])){
row['O'] = "已存在"
row['I'] = 10
@@ -170,6 +207,7 @@ class MapDistrictProcess {
}
}
});
println "6.区域过滤完成"
}
def addFlag(def filePath){
@@ -206,6 +244,8 @@ class MapDistrictProcess {
println index + " " + row
}
});
println "7.打标记完成"
}
def baiduMap(def filePath){
@@ -233,6 +273,7 @@ class MapDistrictProcess {
}
}
});
println "11.特征过滤完成"
}
def mapMatch(def filePath){
@@ -260,16 +301,22 @@ class MapDistrictProcess {
println index +" "+row
},0);
println "4.地图匹配完成"
println "8.地图匹配完成"
}
def houseMatch(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) {
def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
if(rMap){
row['I'] = rMap.flag
}
// def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
// if(rMap){
// row['I'] = rMap.flag
// }
if(row['J']){
if(row['X'] && row['Y'] ){
}
}
}
},0);
}
@@ -314,6 +361,41 @@ class MapDistrictProcess {
},0);
}
def noMatchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && !row['J']) {
if(row['N']>0){
row['I'] = 25
}else{
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 25
}
}
}
});
}
def matchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && row['J'] && row['X'] && row['Y']) {
if(row['X']<=1000 && row['Y']>=0.6){
row['I'] = 22
}else if(row['X']<=600 && row['Y']>=0.5){
row['I'] = 22
}else if(row['X']<=1300 && row['Y']>=0.8){
row['I'] = 22
}else{
row['I'] = 24
}
}else if(!row['I'] && row['J'] && row['Y']==0){
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 26
}
}
});
}
def nameScore(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) {
@@ -343,4 +425,31 @@ class MapDistrictProcess {
}
},0);
}
def updateRegion(){
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx";
ExcelUtils.multiThreadingModify(filePath, { row ->
def region = getRegionByLocation(row['F'], row['G']);
if(region){
row['B'] = RegionData.formatRegion(row['A'], region)
}
},0)
}
def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
if(jsonObject && jsonObject.info=="OK"){
def compent = jsonObject.regeocode.addressComponent
if(compent){
def township = compent.township
return township;
}
}
return null
}
}
@@ -7,7 +7,9 @@ import java.util.regex.Pattern
class NameProcess {
static def filterName(def name){
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心","养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会"]
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心",
"养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房",
"合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼"]
def isAdd = true;
for(def key : keys){
if(name.contains(key)){
@@ -17,7 +19,7 @@ class NameProcess {
}
if(isAdd){
def endKeys = ["路","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍"]
def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门"]
for(def key : endKeys){
if(name.toString().endsWith(key)){
isAdd = false;
@@ -27,7 +29,7 @@ class NameProcess {
}
if(isAdd){
def startKeys = ["FQ"]
def startKeys = ["FQ","社区"]
for(def key : startKeys){
if(name.toString().endsWith(key)||name.toString().startsWith(key)){
isAdd = false;
@@ -36,7 +38,7 @@ class NameProcess {
}
}
if(isAdd){
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房"]
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房", "家属宿舍楼","卫生生活区"]
for(def str : strs){
if(str == name){
isAdd = false;
@@ -56,7 +58,7 @@ class NameProcess {
}
}
//.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号")
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","")
if(name && name.contains("区")){
def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区"))
@@ -191,12 +193,12 @@ class NameProcess {
if(str == name){
return null
}
if(name){
s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
if(s1){
return null
}
}
// if(name){
// s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
// if(s1){
// return null
// }
// }
return name.toUpperCase()
}
@@ -0,0 +1,335 @@
package com.fangjia.script.gj.guangfa
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class GuangfaBlankSelectDistrict {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new GuangfaBlankSelectDistrict();
task.run()
}
def run(){
def file = "D:\\数据\\广发\\获取楼盘信息需求(发送版).xlsx"
def cityMap = standardRegion(file)
def priceMap = getZhiDaoPrice(file)
selectDistricts(cityMap, priceMap)
println "写入完成"
}
def selectDistricts(cityMap, priceMap){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
cityMap.each{city, regions->
if(["西安","东莞","杭州"].contains(city)){
def thirdPriceMap = findNearThirdPartyPrice(city, regions)
def retMaps = findCreditHightDistrict2(city, regions, "2021-08-01", priceMap, thirdPriceMap)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
//基本情况合格的小区
def distMap = [:]
for(def d : districts){
if(regions.contains(d.region)){// && d.household_count && d.household_count>=50
def distKeys = retMaps.get(d.region)
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
if(d.finish_building_date){
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
if(year){
def mod = 2021-year
if(mod <=15){
d.house_year = mod
}else{
d.house_year = null
}
}else{
d.house_year = null
}
}else{
d.house_year = null
}
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// if(d.finish_building_date){
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
// if(year){
// def mod = 2021-year
// if(mod <=15){
// def distKeys = retMaps.get(d.region)
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
// d.house_year = mod
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
// }
// }
// }
}
}
println city +" 基本情况合格的小区:"+distMap.size()
for(def r : regions){
def names = retMaps.get(r)
if(names){
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
rets.addAll(results)
}
}
println "结束"
}
}
println "开始写入Excel....."+rets.size()
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(names.size())
for(def i=0;i<names.size();i++){
def key = names[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def d = distMap.get(key)
if(d){
def ret = getDetailInfo(d, priceMap)
if(ret){
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
// latch.await()
latch.await(5, TimeUnit.MINUTES);
return rets
}
def getDetailInfo(d, priceMap){
def retMap1 = null;
def retMap2 = null;
def retMap3 = null
if(d.city == "上海"){
retMap1 = findShTrades(d, "2021-01-01", "2021-07-01")
retMap2 = findShTrades(d, "2020-01-01", "2020-12-31")
retMap3 = findShTrades(d, "2019-01-01", "2019-12-31")
}else{
retMap1 = CcbCommonUtils.findTrades(d, "2021-01-01", "2021-07-01")
retMap2 = CcbCommonUtils.findTrades(d, "2020-01-01", "2020-12-31")
retMap3 = CcbCommonUtils.findTrades(d, "2019-01-01", "2019-12-31")
}
if(retMap1){
d.count1 = retMap1.count
d.avg_price1 = (int)retMap1.avg_price
}
if(retMap2){
d.count2 = retMap2.count
d.avg_price2 = (int)retMap2.avg_price
}
if(retMap3){
d.count3 = retMap3.count
d.avg_price3 = (int)retMap3.avg_price
}
if(!retMap1 || !retMap2 || !retMap3){
println d.city+" "+ d.region+" "+ d.name +"成交有一年为空过滤"
return null
}
def houses = EsSearchUtils.findSellHouseDetail(d.city, d.region, d.name)
def rooms = []
def areas = []
for(def h : houses){
rooms << h.room
areas << h.area
}
rooms = rooms.unique().sort()
areas = areas.unique().sort()
if(rooms){
d.room = rooms[0]+"-"+rooms[rooms.size()-1]
d.area = areas[0]+"-"+areas[areas.size()-1]
}
def zhidaoprice = priceMap.get(d.city+"#"+d.region)
def ret = [d.city, d.region, zhidaoprice, d.name, d.address, d.developer, d.cubage_percent, d.estate_company, d.household_count, d.house_year,d.room, d.area, d.count1, d.avg_price1,d.count2, d.avg_price2,d.count3, d.avg_price3]
println ret
return ret;
}
def standardRegion(filePath){
def city = null
def cityMap = new LinkedHashMap()
def index = 0
ExcelUtils.modify(filePath, { row ->
index ++
if(index >= 4){
if(row['C']){
city = row['C']
}
println city+" "+row['D']
def newRegion = null
if(city && row['D']){
newRegion = RegionData.formatRegion(city, row['D'])
row['F'] = newRegion
}
if(city && newRegion){
def regions = cityMap.get(city)
if(!regions){
regions = []
cityMap.put(city, regions)
}
if(!regions.contains(newRegion)){
regions << newRegion
}
}
}
});
return cityMap
}
def findCreditHightDistrict2(def city, def regions, def weekend, def priceMap, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
q.put("region", new BasicDBObject('$in', regions))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
if(regions.contains(d.region)){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice){
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def price = priceMap.get(d.city +"#"+d.region)
if(d.avg_price>price){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}else{
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}
return retMaps;
}
def getZhiDaoPrice(filePath){
def priceMap = [:]
def city = null
ExcelUtils.modify(filePath, { row ->
if(row['C']){
city = row['C']
}
if(city && row['F']){
def key = city +"#"+row['F']
if(!priceMap.containsKey(key)){
priceMap.put(key, row['E'])
}
}
})
return priceMap
}
public static def findShTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("fjCity", dist.city)
q.put("fjRegion", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("fjName", new BasicDBObject('$in', dist.alias))
}else{
q.put("fjName", dist.name)
}
q.put("contractTime", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = youda_esfhouse_coll.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.price){
total_price = total_price + d.price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
def findNearThirdPartyPrice(def city, regions){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", new BasicDBObject('$in', regions))
q.put("s_date", new BasicDBObject('$gte', 202105))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
}
@@ -24,21 +24,22 @@ class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
Date date=new Date(1622802609);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String str = format.format(date);
System.out.println(str);
// Date date=new Date(1622802609);
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// String str = format.format(date);
// System.out.println(str);
// def task = new LocationTest()
def task = new LocationTest()
// task.run()
// task.run2()
// task.run3()
task.run3()
// task.run4()
}
def run(){
def regionPointsMap = findRegionRange()
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
def city = "南昌"
def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0
def rets = []
for(def dist : districts){
@@ -74,7 +75,7 @@ class LocationTest {
}
}
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内")
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内")
}
def run2(){
@@ -133,10 +134,10 @@ class LocationTest {
def run3(){
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d->
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d->
def distance = null
if(d['G']){
distance = d['G'].toDouble()
if(d['H']){
distance = d['H'].toDouble()
}
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
@@ -180,10 +181,56 @@ class LocationTest {
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
def dbo = DbUtils.seaweed.findOne(q)
d['K'] = "修正"
}
}else if(d['E'] && d['B'] != d['E'] && distance && distance<=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
if(mapName.contains("-") && mapName.endsWith("楼")){
mapName = mapName.substring(0, mapName.indexOf("-"))
}
if(d['C']==mapName){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['E'])
q.put("name", d['C'])
q.put("cat", 'district')
q.put("status", 0)
def dbo = DbUtils.seaweed.findOne(q)
if(!dbo){
q.put("region", d['B'])
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(d['D']=="高德"){
lng = (d['I'].toDouble()*10000000000).longValue()
lat = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
}else{
lng2 = (d['I'].toDouble()*10000000000).longValue()
lat2 = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng = (gmap.get("lng")*10000000000).longValue()
lat = (gmap.get("lat")*10000000000).longValue()
}
}
def udata = new BasicDBObject()
udata.put("region", d['E'])
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
d['K'] = "修正区域"
}
}
}
},0)
@@ -366,13 +413,13 @@ class LocationTest {
return flag;
}
def findRegionRange(){
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city","福州市")
q.put("city",city+"市")
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion("福州",d.region)
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
}
return regionMap
@@ -531,8 +531,10 @@ public class DataUtil {
if(!(address ==~ /.*(½»»ã|½»¿Ú|½»²æ¿Ú)$/)){
address += "½»»ã"
}
}else{
address = null
}
address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
// address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
}
return address
}