Merge branch 'script_gj' into 'master'

update

See merge request !66
This commit is contained in:
2021-08-03 19:17:17 +08:00
20 changed files with 931 additions and 228 deletions
@@ -1,6 +1,7 @@
package com.fangjia.script.ccbevaluation package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
@@ -40,12 +41,11 @@ class BigdataCommunityBatch extends BaseBatch{
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", "总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] "乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){ for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists)
rows.addAll(rets) rows.addAll(rets)
println city +" "+rets.size() println city +" "+rets.size()
@@ -57,10 +57,10 @@ class BigdataCommunityBatch extends BaseBatch{
allDatas.addAll(rows) allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false); FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false);
println "txt 生成" println "txt 生成"
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
} }
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){ def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
def rets = Collections.synchronizedList(new ArrayList()); def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size()) final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -70,7 +70,7 @@ class BigdataCommunityBatch extends BaseBatch{
void run() { void run() {
try{ try{
totalCount.addAndGet(1) totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, distCodeMap, dist) def ret = createData(cityCodeDataMap, distCodeMap, dist)
if(ret){ if(ret){
rets << ret rets << ret
} }
@@ -87,10 +87,10 @@ class BigdataCommunityBatch extends BaseBatch{
return rets; return rets;
} }
def createData(def cityCodeDatas, def distCodeMap, def dist){ def createData(def cityCodeDataMap, def distCodeMap, def dist){
def ret = [] def ret = []
try{ try{
ret << cityCodeDatas.city_code ret << cityCodeDataMap.get(dist.city)
if(dist.region){ if(dist.region){
ret << distCodeMap.get(dist.region) ret << distCodeMap.get(dist.region)
}else{ }else{
@@ -101,9 +101,17 @@ class BigdataCommunityBatch extends BaseBatch{
ret << dist.region ret << dist.region
ret << dist.name ret << dist.name
ret << dist.address ret << dist.address
if(dist.lat){ if(dist.lat2){
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6) def lat = dist.lat2/10000000000d
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6) def lng = dist.lng2/10000000000d
def loactionMap = GeoUtils.bd09ToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{ }else{
ret << "" ret << ""
ret << "" ret << ""
@@ -120,7 +128,7 @@ class BigdataCommunityBatch extends BaseBatch{
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << dist.status ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "物业小区" ret << "pa100"
ret << "" ret << ""
if(dist.address){ if(dist.address){
if(dist.address.toString().endsWith("弄")){ if(dist.address.toString().endsWith("弄")){
@@ -27,8 +27,8 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{ class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -44,35 +44,43 @@ class BigdataDealBatch extends BaseBatch{
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
def rows = [] def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){ for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def trades = getCityData(city) def trades = getCityData(city)
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other") def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets) rows.addAll(rets)
def youDataTrade = getYouDaData() println city +" 成交共:"+rets.size()
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda") if(city=="上海"){
rows.addAll(youdaDatas) def youDataTrade = getYouDaData()
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
println city +" "+youdaDatas.size()
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); // FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = [] def cityDatas = []
cityDatas << headers cityDatas << headers
cityDatas.addAll(rets) cityDatas.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
} }
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false); def fileName = "/home/goojia/ccb/bigdata_deal.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成" println "txt 生成"
def rets = [] // def rets = []
rets << headers // rets << headers
rets.addAll(rows) // rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据") // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
} }
def createData(def rets, cityCodeDatas, distCodeMap, source){ def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -114,9 +122,9 @@ class BigdataDealBatch extends BaseBatch{
if(!obj){ if(!obj){
return ret; return ret;
} }
ret << cityCodeDatas.city_code ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString() ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city ret << trade.city
if(trade.fjRegion){ if(trade.fjRegion){
ret << distCodeMap.get(trade.fjRegion) ret << distCodeMap.get(trade.fjRegion)
}else{ }else{
@@ -215,11 +223,11 @@ class BigdataDealBatch extends BaseBatch{
ret << CcbCodeUtils.getFitmentCode(trade.fitment) ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << "" ret << ""
ret << trade.price ret << trade.price
ret << trade.totalPrice ret << trade.totalPrice * 10000
ret << "" ret << ""
ret << "" ret << ""
ret << "" ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "" ret << ""
ret << "" ret << ""
ret << "" ret << ""
@@ -240,7 +248,7 @@ class BigdataDealBatch extends BaseBatch{
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW" ret << "FJW"
ret << trade.totalPrice ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret; return ret;
} }
@@ -248,9 +256,9 @@ class BigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){ def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = [] def ret = []
ret << cityCodeDatas.city_code ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString() ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city ret << trade.city
if(trade.region){ if(trade.region){
ret << distCodeMap.get(trade.region) ret << distCodeMap.get(trade.region)
}else{ }else{
@@ -317,7 +325,7 @@ class BigdataDealBatch extends BaseBatch{
ret << "" ret << ""
ret << "" ret << ""
ret << "" ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
ret << "" ret << ""
ret << "" ret << ""
ret << "" ret << ""
@@ -373,15 +381,23 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size()) final CountDownLatch latch = new CountDownLatch(dists.size())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = null;
if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
}else{
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
}
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){ for(def i=0;i<dists.size();i++){
final def dist = dists[i] final def dist = dists[i]
fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
void run() { void run() {
try { try {
totalCount.addAndGet(1) totalCount.addAndGet(1)
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date())) def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate)
if(trades){ if(trades){
tradesList.addAll(trades) tradesList.addAll(trades)
} }
@@ -12,6 +12,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject import com.mongodb.DBObject
import groovy.sql.Sql
import org.bson.types.ObjectId import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
@@ -54,7 +55,8 @@ class BigdataForsaleBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) { static main(args) {
runBatch(new BigdataForsaleBatch(), args); runBatch(new BigdataForsaleBatch(), args);
@@ -63,15 +65,19 @@ class BigdataForsaleBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
def index = 0
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){ for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) index++
def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, distIds,20200701, 20210731);
} }
} }
def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){ def exportSellCSV(index,city, cityCodeDataMap, distCodeMap,districts, distIds,startDate, endDate){
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size()) final CountDownLatch latch = new CountDownLatch(districts.size())
@@ -85,36 +91,42 @@ class BigdataForsaleBatch extends BaseBatch{
void run() { void run() {
try { try {
totalCount.addAndGet(1) totalCount.addAndGet(1)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) def commId = rMap.get("_id").toString()
if(ret){ if(distIds.contains(commId)){
def cityCode = cityCodeDatas.city_code def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
def regionCode = distCodeMap.get(rMap.region) if(ret && ret.size()<100){
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = rMap.region
m.region_code = regionCode
m.district_id = rMap.get("_id").toString()
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
} }
if(ret){
def cityCode = cityCodeDataMap.get(city)
def regionCode = distCodeMap.get(rMap.region)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = rMap.region
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
}
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
} }
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace() e.printStackTrace()
} finally { } finally {
@@ -124,14 +136,27 @@ class BigdataForsaleBatch extends BaseBatch{
}) })
} }
latch.await() latch.await()
def fileName = "/home/goojia/ccb/bigdata_forsale.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt"
}
println city + " 房源总量:"+retMap.size() println city + " 房源总量:"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values()); // def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values() def allDatas = retMap.values()
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] def headers = []
if(index == 1){
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true); headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
}
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 " println "txt文件生成 "
// def rets = []
// rets << headers
// rets.addAll(allDatas)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据")
} }
@@ -236,6 +261,15 @@ class BigdataForsaleBatch extends BaseBatch{
alias.add(name) alias.add(name)
} }
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){ if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){ for(def s : alias){
@@ -358,8 +392,9 @@ class BigdataForsaleBatch extends BaseBatch{
if(seawater.p_date instanceof String){ if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger() seawater.p_date = seawater.p_date.toInteger()
} }
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ if(seawater.p_date<=20200101){
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") //seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date rowMap.m_date = rowMap.p_date
}else{ }else{
@@ -518,4 +553,14 @@ class BigdataForsaleBatch extends BaseBatch{
println ids.size() println ids.size()
} }
def getDistrictIds(def cityid){
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
def distIds = new HashSet()
sqlInstance.eachRow(sql,{it->
distIds << it.commid
});
return distIds;
}
} }
@@ -7,7 +7,7 @@ import com.mongodb.DBCollection
class CcbCommonUtils { class CcbCommonUtils {
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] public static def citys = ["成都","沈阳","大连","北京", "上海","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","重庆","南京","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"] // public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -23,21 +23,10 @@ class CcbCommonUtils {
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def findCityCodeValue(city){
def q = new BasicDBObject() public static def findCityCodeValue(){
q.put("fangjia_city", city) def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) return cityCodeMap;
def count = dbcursor.count()
if(count >1){
def data = null
for(def db : dbcursor){
if(db.city_code){
data = db
break
}
}
return data;
}
} }
public static def findAllDistrictCodeValue(city){ public static def findAllDistrictCodeValue(city){
@@ -105,4 +94,45 @@ class CcbCommonUtils {
return lists return lists
} }
public static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
} }
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
results << headers results << headers
results.addAll(rows) results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据") ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8"); FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\staging_district.txt",headers, "utf-8", false);
} }
@@ -76,9 +76,9 @@ class StagingDistrictBatch extends BaseBatch{
def getFenQiMap(def dists){ def getFenQiMap(def dists){
def fenQiMap = [:] def fenQiMap = [:]
for(def d : dists){ for(def d : dists){
if(d.master_district_name){ // if(d.master_district_name){
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d) fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
} // }
} }
return fenQiMap; return fenQiMap;
} }
@@ -19,7 +19,7 @@ import com.mongodb.DBCollection
class ThirdPartyPriceBatch extends BaseBatch { class ThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -30,7 +30,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = CcbCommonUtils.citys; def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]//CcbCommonUtils.citys;
def rows = [] def rows = []
def index = 0 def index = 0
for(def city : citys){ for(def city : citys){
@@ -166,7 +166,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
def dbcursor = AVERAGE_STATISTICS.find(q) def dbcursor = AVERAGE_STATISTICS.find(q)
def map = [:] def map = [:]
for(def db : dbcursor){ for(def db : dbcursor){
map.put(db.region+"##"+db.name, db) if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
map.put(db.region+"##"+db.name, db)
}
} }
return map return map
} }
@@ -698,6 +698,28 @@ class CommonDatautils {
return rets; return rets;
} }
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", new BasicDBObject('$in', regions))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllDistrictByQuery(def q, def cat, def fields){ public static def findAllDistrictByQuery(def q, def cat, def fields){
@@ -2,6 +2,8 @@ package com.fangjia.script.gj
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DataUtil
@@ -63,7 +65,7 @@ class AddThirdPriceDistrict {
def resAddrMap = [:] def resAddrMap = [:]
def filterEstateTypeMap = [:] def filterEstateTypeMap = [:]
public static EsDataSource dataSource = new EsDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -82,14 +84,77 @@ class AddThirdPriceDistrict {
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") // println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
} }
def run(){ def run(){
exportDatas(); // exportDatas();
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); // getDataType("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addInfo(); // addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx") // addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx") // addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx")
addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); // addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
} }
def dataFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
def name = row['C']
if(name){
def flag = NameProcess.filterName(name)
if(!flag){
row['T'] = 1
}else{
if(name.toString().length()<3){
if(row['N'] && row['N']>=10 && !row['J']){
row['I'] = 25
}else{
row['I'] = 18
}
}else{
def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){
row['I'] = 19
}
}
}
}else{
if(!name){
row['I'] = 1
}
}
});
println "3.名称过滤完成"
}
def addNewMapInfo(def path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['I']){
def gdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "高德")
if(gdData){
row['F'] = gdData.district
row['G'] = gdData.name
row['H'] = gdData.address
row['I'] = gdData.type
row['J'] = gdData.score
row['K'] = gdData.lng
row['L'] = gdData.lat
}
def bdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "百度")
if(bdData){
row['M'] = bdData.district
row['N'] = bdData.name
row['O'] = bdData.address
row['P'] = bdData.type
row['Q'] = bdData.score
row['R'] = bdData.lng
row['S'] = bdData.lat
}
}
},0);
}
def addCrawlMapInfo( def path){ def addCrawlMapInfo( def path){
ExcelUtils.multiThreadingModify(path, { row -> ExcelUtils.multiThreadingModify(path, { row ->
def flag = row['R'] def flag = row['R']
@@ -105,27 +170,27 @@ class AddThirdPriceDistrict {
row['K'] = data.lat row['K'] = data.lat
} }
def baiduRets = findBaiduMap(row['A'], row['B'], row['C']) // def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
def bdRegion =null // def bdRegion =null
def bdName = null // def bdName = null
def bdAddress = null // def bdAddress = null
def bdLng = null // def bdLng = null
def bdLat = null // def bdLat = null
def bdEstateType = null // def bdEstateType = null
if(baiduRets){ // if(baiduRets){
bdRegion = baiduRets.region // bdRegion = baiduRets.region
bdName = baiduRets.name // bdName = baiduRets.name
bdAddress = baiduRets.address // bdAddress = baiduRets.address
bdLng = baiduRets.lng // bdLng = baiduRets.lng
bdLat = baiduRets.lat // bdLat = baiduRets.lat
bdEstateType = baiduRets.estateType // bdEstateType = baiduRets.estateType
} // }
row['L'] = bdRegion // row['L'] = bdRegion
row['M'] = bdName // row['M'] = bdName
row['N'] = bdAddress // row['N'] = bdAddress
row['O'] = bdEstateType // row['O'] = bdEstateType
row['P'] = bdLng // row['P'] = bdLng
row['Q'] = bdLat // row['Q'] = bdLat
} }
},0); },0);
@@ -502,8 +567,8 @@ class AddThirdPriceDistrict {
return thirdPartyColl.findOne(q) return thirdPartyColl.findOne(q)
} }
def addInfo(){ def addInfo(filePath){
province = "江西"
// def city = "苏州" // def city = "苏州"
// def datas = findMatchErrorDatas(city); // def datas = findMatchErrorDatas(city);
// println "共【"+datas.size()+"】条" // println "共【"+datas.size()+"】条"
@@ -512,7 +577,7 @@ class AddThirdPriceDistrict {
// def resMap = groupByName(datas); // def resMap = groupByName(datas);
def index = 0; def index = 0;
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx", { row -> ExcelUtils.multiThreadingModify(filePath, { row ->
def flag = row['R'] def flag = row['R']
def dgDatas = row['G'] def dgDatas = row['G']
// if(!flag){ // if(!flag){
@@ -956,11 +1021,11 @@ class AddThirdPriceDistrict {
def queryPoiMap(def city, def region, def name){ def queryPoiMap(def city, def region, def name){
final StringBuffer sqlBuffer = new StringBuffer() final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1") sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'") sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and adname = '" + region + "'") sqlBuffer.append(" and adname = '" + region + "'")
sqlBuffer.append(" and name = '" + name + "'") sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and source = '高德'") sqlBuffer.append(" and source = '高德' order by score() desc limit 1")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def map = [:] def map = [:]
while (rs.next()) { while (rs.next()) {
@@ -969,8 +1034,11 @@ class AddThirdPriceDistrict {
map.name = rs.getString("name") map.name = rs.getString("name")
map.address = rs.getString("address") map.address = rs.getString("address")
map.type = rs.getString("type") map.type = rs.getString("type")
map.lng = rs.getString("location.lon") def location = rs.getObject("location")
map.lat = rs.getString("location.lat") if(location){
map.lng = location.x
map.lat = location.y
}
} }
return map; return map;
} }
@@ -64,24 +64,29 @@ class EsSearchUtils {
sqlBuffer.append(" and region = '"+region+"'") sqlBuffer.append(" and region = '"+region+"'")
sqlBuffer.append(" and district_name = '"+name+"'") sqlBuffer.append(" and district_name = '"+name+"'")
sqlBuffer.append(" and cat = 'sell'") sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" limit 100") sqlBuffer.append(" limit 200")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def rets = [] def rets = []
while (rs.next()){ while (rs.next()){
def height = rs.getInt("height")
def direction = rs.getString("direction")
def fitment = rs.getString("fitment")
def room = rs.getInt("room") def room = rs.getInt("room")
def hall = rs.getInt("hall") if(room){
def toilet = rs.getInt("toilet") def height = rs.getInt("height")
def map = [:] def direction = rs.getString("direction")
map.height = height def fitment = rs.getString("fitment")
map.direction = direction def hall = rs.getInt("hall")
map.fitment = fitment def toilet = rs.getInt("toilet")
map.room = room def area = rs.getInt("area")
map.hall = hall def map = [:]
map.toilet = toilet map.height = height
rets << map map.direction = direction
map.fitment = fitment
map.room = room
map.hall = hall
map.toilet = toilet
map.area = area
rets << map
}
} }
return rets; return rets;
} }
@@ -50,14 +50,14 @@ class GaoDeMapCrawl extends BaseBatch{
// def categorys = ["141202", "141203"]//中学,小学 // def categorys = ["141202", "141203"]//中学,小学
// def prefix = "district_" def prefix = "district_"
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def prefix = "road_" // def prefix = "road_"
// def categorys = ["190301"]//道路相关 // def categorys = ["190301"]//道路相关
def prefix = "village_" // def prefix = "village_"
def categorys = ["190108","190109"]//村庄 // def categorys = ["190108","190109"]//村庄
// def prefix = "company_" // def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] // def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
@@ -119,7 +119,7 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆" def run(){//,"西安","重庆"
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys =["无锡"] def citys =["南昌"]
// def citys = CommonUtils.findAllCity() // def citys = CommonUtils.findAllCity()
def totalCount = 0; def totalCount = 0;
for(def city : citys){ for(def city : citys){
@@ -28,8 +28,8 @@ class RegionPointCrawl {
static main(args) { static main(args) {
def task = new RegionPointCrawl(); def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"] // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] def provinces = ["江西省"]
for(def p : provinces){ for(def p : provinces){
task.crawlCityAndDistrictCode(p); task.crawlCityAndDistrictCode(p);
task.run(p); task.run(p);
@@ -72,7 +72,7 @@ class RegionPointCrawl {
for(def cityMap : cityList){ for(def cityMap : cityList){
if(cityMap.level == "city"){ if(cityMap.level == "city"){
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level); saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
saveGaoDeRegionCode(province, cityMap.districts); saveGaoDeRegionCode(province, cityMap.name, cityMap.districts);
} }
} }
} }
@@ -81,10 +81,10 @@ class RegionPointCrawl {
} }
} }
def saveGaoDeRegionCode(province, districts){ def saveGaoDeRegionCode(province, cityName,districts){
if(districts){ if(districts){
for(def dist : districts){ for(def dist : districts){
if(dist.level=="district"){ if(dist.level=="district" || cityName=="东莞市"){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
} }
} }
@@ -221,4 +221,3 @@ class RegionPointCrawl {
return KeyList.get(count); return KeyList.get(count);
} }
} }
H
@@ -12,8 +12,10 @@ class FileUtils {
OutputStream out = new FileOutputStream(fileName, isAppend); OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer(); def sb = new StringBuffer();
sb.append(headers.join("|@|")) if(headers){
sb.append(System.getProperty("line.separator")); sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
}
out.write(sb.toString().getBytes(charset)); out.write(sb.toString().getBytes(charset));
sb.setLength(0) sb.setLength(0)
@@ -15,7 +15,19 @@ class BestDistrictPriceExport {
static main(args) { static main(args) {
def task = new BestDistrictPriceExport(); def task = new BestDistrictPriceExport();
task.run() //task.run()
task.run2()
}
def run2(){
def filePath = "d:\\数据\\广发\\楼盘.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
def stat = findDistrictPrice(row['A'], row['B'], row['D'], "20210801")
if(stat){
row['S'] = stat.avg_price
}
println row['A']+" "+row['B']+" "+ row['C']+" "+row['S']
})
} }
def run(){ def run(){
@@ -24,10 +24,10 @@ class DataProcess {
static main(args) { static main(args) {
//建行信用卡城市(临汾--晋城).xlsx //建行信用卡城市(临汾--晋城).xlsx
def filePath = "D:\\数据\\建行银行\\建行信用卡入库小区V1.xlsx" def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx"
def task = new DataProcess(); def task = new DataProcess();
// task.batchSaveDataToDB(filePath) // task.batchSaveDataToDB(filePath)
// task.mapBatchSaveDataToDB(filePath) task.mapBatchSaveDataToDB(filePath)
// task.calcutePrice(filePath); // task.calcutePrice(filePath);
// task.calcutePrice2(); // task.calcutePrice2();
// task.deleteRepDist("株洲"); // task.deleteRepDist("株洲");
@@ -35,7 +35,7 @@ class DataProcess {
// task.standartName(filePath); // task.standartName(filePath);
task.saveGaoDeMap(filePath); // task.saveGaoDeMap(filePath);
// task.updateDBAddress(); // task.updateDBAddress();
// task.updateRegion() // task.updateRegion()
@@ -120,13 +120,10 @@ class DataProcess {
def lastDatas = [] def lastDatas = []
if(datas){ if(datas){
def keys = [] def keys = []
for(def data : datas){ for(def data : datas){//20,21,25,
if(data['A']=="厦门" && data['I'] && data['I'].toString().startsWith("24")){ if(data['I'] && [26,27].contains(data['I'].toString().toDouble().toInteger())){
def key = data['A']+"##"+data['B']+"##"+data['H'] def key = data['A']+"##"+data['B']+"##"+data['H']
if(!keys.contains(key) && data['H'].size()>2){ if(!keys.contains(key) && data['H'] && data['H'].size()>2){
// if(data['I'].toInteger()==22){//&& data['N']<3
// continue;
// }
keys << key keys << key
lastDatas << data lastDatas << data
} }
@@ -168,7 +165,7 @@ class DataProcess {
addIndex++ addIndex++
def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null) def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null)
println "${addIndex}【新增入库】"+ values println "${addIndex}【新增入库】"+ values
// FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4); FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
}else{ }else{
println "【已存在】 ${d['A']} ${d['B']} ${d['H']}" println "【已存在】 ${d['A']} ${d['B']} ${d['H']}"
} }
@@ -5,6 +5,7 @@ import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.EsDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
@@ -26,8 +27,9 @@ class ExportData {
static main(args) { static main(args) {
def task = new ExportData(); def task = new ExportData();
// task.exportThird() task.exportThird()
task.exportMap(); // task.exportMap();
} }
def exportMap(){ def exportMap(){
@@ -72,7 +74,7 @@ class ExportData {
def exportThird(){ def exportThird(){
def citys = ["重庆"] def citys = ["南昌"]
for(def city : citys){ for(def city : citys){
def rets = [] def rets = []
rets << ["城市","区域","名称","别名"] rets << ["城市","区域","名称","别名"]
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
@@ -7,6 +8,7 @@ import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import java.util.regex.Pattern import java.util.regex.Pattern
@@ -15,32 +17,39 @@ class MapDistrictProcess {
static main(args) { static main(args) {
def task = new MapDistrictProcess(); def task = new MapDistrictProcess();
task.run() task.run()
// task.updateRegion()
} }
/** /**
* 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路 * 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路
* 20:不同区域新增(库中无) 21:分期新增 22:分数在0.6以上 * 20:不同区域新增(库中无) 21:分期新增 23:不同区域,距离相近 25:无匹配,可新增 22:有匹配,距离1000米以下,相似度在0.6以上 24:疑似匹配有误 27:人工判定入库
* @return * @return
*/ */
def run(){ def run(){
def files = ["武汉.xlsx"] // def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
def files = ["东莞.xlsx"]
for(def fileName : files){ for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName; def filePath = "D:\\数据\\小区\\地图\\"+fileName;
// typeFilter(filePath) typeFilter(filePath)
// nameStandard(filePath); nameStandard(filePath);
// dataFilter(filePath) dataFilter(filePath)
matchDistDatas(filePath)
houseCalculate(filePath); houseCalculate(filePath);
////// //统计是否存在多个区域 ////// //统计是否存在多个区域
matchDistDatas(filePath)
moreRegionCalculate(filePath) moreRegionCalculate(filePath)
addFlag(filePath) addFlag(filePath)
mapMatch(filePath); mapMatch(filePath);
baiduTypeFilter(filePath); baiduTypeFilter(filePath);
baiduMap(filePath); // baiduMap(filePath);
distanceFilter(filePath)
predictType(filePath)
noMatchFlag(filePath)
matchFlag(filePath)
addFeature(filePath) addFeature(filePath)
houseMatch(filePath); houseMatch(filePath);
predictType(filePath)
// nameScore(filePath); // nameScore(filePath);
} }
@@ -55,7 +64,12 @@ class MapDistrictProcess {
row['I'] = 1 row['I'] = 1
}else{ }else{
if(name.toString().length()<3){ if(name.toString().length()<3){
row['I'] = 18 if(row['N'] && row['N']>=10 && !row['J']){
row['I'] = 25
}else{
row['I'] = 18
}
}else{ }else{
def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$")) def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){ if(s1){
@@ -70,7 +84,7 @@ class MapDistrictProcess {
} }
}); });
println "2.名称过滤完成" println "3.名称过滤完成"
} }
def typeFilter(filePath){ def typeFilter(filePath){
@@ -84,15 +98,30 @@ class MapDistrictProcess {
println "1.类型过滤完成" println "1.类型过滤完成"
} }
def baiduTypeFilter(filePath){ def baiduTypeFilter(filePath){
def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"] def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"]
ExcelUtils.multiThreadingModify(filePath, { row -> ExcelUtils.multiThreadingModify(filePath, { row ->
def type = row['S'] def type = row['S']
if(type && !types.contains(type)){ if(type && row['Q'] && row['Q']==row['H'] && !types.contains(type)){
row['I'] = 1 row['I'] = 1
} }
}); });
println "1.类型过滤完成" println "9.百度类型过滤完成"
}
def distanceFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']){
if(row['X'] && row['Y']){
if(row['X']<=500 && row['Y']>=0.5){
row['I'] = 12
}
}
}
});
println "10.距离过滤完成"
} }
def nameStandard(filePath){ def nameStandard(filePath){
@@ -105,12 +134,12 @@ class MapDistrictProcess {
} }
}, 0); }, 0);
println "1.名称标准化完成" println "2.名称标准化完成"
} }
def matchDistDatas(def path){ def matchDistDatas(def path){
def index = 0 def index = 0
ExcelUtils.modify(path,{ List row -> ExcelUtils.multiThreadingModify(path,{ List row ->
index ++ index ++
if(!row['I'] && !row['J']){ if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']); def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
@@ -126,13 +155,16 @@ class MapDistrictProcess {
def maplat = baidu.y def maplat = baidu.y
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng) row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L']) def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
if(score<0.3){
score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['M'])
}
row['Y'] = score row['Y'] = score
} }
} }
println index +" "+row println index +" "+row
} }
},0) },0)
println "3.小区匹配完成" println "5.小区匹配完成"
} }
def houseCalculate(filePath){ def houseCalculate(filePath){
@@ -141,6 +173,7 @@ class MapDistrictProcess {
row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H']) row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H'])
} }
}); });
println "4.房源统计完成"
} }
def moreRegionCalculate(filePath){ def moreRegionCalculate(filePath){
@@ -161,8 +194,12 @@ class MapDistrictProcess {
} }
} }
if(!regions.contains(row['B'])){ if(!regions.contains(row['B'])){
row['O'] = regions.toString() if(row['X'] && row['X']<=500 && row['Y'] && row['Y']>=0.5){
row['I'] = 20 row['I'] = 23
}else{
row['O'] = regions.toString()
row['I'] = 20
}
}else if(regions.contains(row['B'])){ }else if(regions.contains(row['B'])){
row['O'] = "已存在" row['O'] = "已存在"
row['I'] = 10 row['I'] = 10
@@ -170,6 +207,7 @@ class MapDistrictProcess {
} }
} }
}); });
println "6.区域过滤完成"
} }
def addFlag(def filePath){ def addFlag(def filePath){
@@ -206,6 +244,8 @@ class MapDistrictProcess {
println index + " " + row println index + " " + row
} }
}); });
println "7.打标记完成"
} }
def baiduMap(def filePath){ def baiduMap(def filePath){
@@ -233,6 +273,7 @@ class MapDistrictProcess {
} }
} }
}); });
println "11.特征过滤完成"
} }
def mapMatch(def filePath){ def mapMatch(def filePath){
@@ -260,16 +301,22 @@ class MapDistrictProcess {
println index +" "+row println index +" "+row
},0); },0);
println "4.地图匹配完成" println "8.地图匹配完成"
} }
def houseMatch(filePath){ def houseMatch(filePath){
ExcelUtils.multiThreadingModify(filePath, { row -> ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) { if(!row['I']) {
def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N']) // def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
if(rMap){ // if(rMap){
row['I'] = rMap.flag // row['I'] = rMap.flag
} // }
if(row['J']){
if(row['X'] && row['Y'] ){
}
}
} }
},0); },0);
} }
@@ -314,6 +361,41 @@ class MapDistrictProcess {
},0); },0);
} }
def noMatchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && !row['J']) {
if(row['N']>0){
row['I'] = 25
}else{
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 25
}
}
}
});
}
def matchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && row['J'] && row['X'] && row['Y']) {
if(row['X']<=1000 && row['Y']>=0.6){
row['I'] = 22
}else if(row['X']<=600 && row['Y']>=0.5){
row['I'] = 22
}else if(row['X']<=1300 && row['Y']>=0.8){
row['I'] = 22
}else{
row['I'] = 24
}
}else if(!row['I'] && row['J'] && row['Y']==0){
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 26
}
}
});
}
def nameScore(filePath){ def nameScore(filePath){
ExcelUtils.multiThreadingModify(filePath, { row -> ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) { if(!row['I']) {
@@ -343,4 +425,31 @@ class MapDistrictProcess {
} }
},0); },0);
} }
def updateRegion(){
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx";
ExcelUtils.multiThreadingModify(filePath, { row ->
def region = getRegionByLocation(row['F'], row['G']);
if(region){
row['B'] = RegionData.formatRegion(row['A'], region)
}
},0)
}
def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
if(jsonObject && jsonObject.info=="OK"){
def compent = jsonObject.regeocode.addressComponent
if(compent){
def township = compent.township
return township;
}
}
return null
}
} }
@@ -7,7 +7,9 @@ import java.util.regex.Pattern
class NameProcess { class NameProcess {
static def filterName(def name){ static def filterName(def name){
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心","养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会"] def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心",
"养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房",
"合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼"]
def isAdd = true; def isAdd = true;
for(def key : keys){ for(def key : keys){
if(name.contains(key)){ if(name.contains(key)){
@@ -17,7 +19,7 @@ class NameProcess {
} }
if(isAdd){ if(isAdd){
def endKeys = ["路","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍"] def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门"]
for(def key : endKeys){ for(def key : endKeys){
if(name.toString().endsWith(key)){ if(name.toString().endsWith(key)){
isAdd = false; isAdd = false;
@@ -27,7 +29,7 @@ class NameProcess {
} }
if(isAdd){ if(isAdd){
def startKeys = ["FQ"] def startKeys = ["FQ","社区"]
for(def key : startKeys){ for(def key : startKeys){
if(name.toString().endsWith(key)||name.toString().startsWith(key)){ if(name.toString().endsWith(key)||name.toString().startsWith(key)){
isAdd = false; isAdd = false;
@@ -36,7 +38,7 @@ class NameProcess {
} }
} }
if(isAdd){ if(isAdd){
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房"] def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房", "家属宿舍楼","卫生生活区"]
for(def str : strs){ for(def str : strs){
if(str == name){ if(str == name){
isAdd = false; isAdd = false;
@@ -56,7 +58,7 @@ class NameProcess {
} }
} }
//.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号") //.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号")
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim() name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","") name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","")
if(name && name.contains("区")){ if(name && name.contains("区")){
def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区")) def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区"))
@@ -191,12 +193,12 @@ class NameProcess {
if(str == name){ if(str == name){
return null return null
} }
if(name){ // if(name){
s1 = DataUtil.match(name, Pattern.compile("^\\w+")) // s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
if(s1){ // if(s1){
return null // return null
} // }
} // }
return name.toUpperCase() return name.toUpperCase()
} }
@@ -0,0 +1,335 @@
package com.fangjia.script.gj.guangfa
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class GuangfaBlankSelectDistrict {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new GuangfaBlankSelectDistrict();
task.run()
}
def run(){
def file = "D:\\数据\\广发\\获取楼盘信息需求(发送版).xlsx"
def cityMap = standardRegion(file)
def priceMap = getZhiDaoPrice(file)
selectDistricts(cityMap, priceMap)
println "写入完成"
}
def selectDistricts(cityMap, priceMap){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
cityMap.each{city, regions->
if(["西安","东莞","杭州"].contains(city)){
def thirdPriceMap = findNearThirdPartyPrice(city, regions)
def retMaps = findCreditHightDistrict2(city, regions, "2021-08-01", priceMap, thirdPriceMap)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
//基本情况合格的小区
def distMap = [:]
for(def d : districts){
if(regions.contains(d.region)){// && d.household_count && d.household_count>=50
def distKeys = retMaps.get(d.region)
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
if(d.finish_building_date){
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
if(year){
def mod = 2021-year
if(mod <=15){
d.house_year = mod
}else{
d.house_year = null
}
}else{
d.house_year = null
}
}else{
d.house_year = null
}
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// if(d.finish_building_date){
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
// if(year){
// def mod = 2021-year
// if(mod <=15){
// def distKeys = retMaps.get(d.region)
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
// d.house_year = mod
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
// }
// }
// }
}
}
println city +" 基本情况合格的小区:"+distMap.size()
for(def r : regions){
def names = retMaps.get(r)
if(names){
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
rets.addAll(results)
}
}
println "结束"
}
}
println "开始写入Excel....."+rets.size()
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(names.size())
for(def i=0;i<names.size();i++){
def key = names[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def d = distMap.get(key)
if(d){
def ret = getDetailInfo(d, priceMap)
if(ret){
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
// latch.await()
latch.await(5, TimeUnit.MINUTES);
return rets
}
def getDetailInfo(d, priceMap){
def retMap1 = null;
def retMap2 = null;
def retMap3 = null
if(d.city == "上海"){
retMap1 = findShTrades(d, "2021-01-01", "2021-07-01")
retMap2 = findShTrades(d, "2020-01-01", "2020-12-31")
retMap3 = findShTrades(d, "2019-01-01", "2019-12-31")
}else{
retMap1 = CcbCommonUtils.findTrades(d, "2021-01-01", "2021-07-01")
retMap2 = CcbCommonUtils.findTrades(d, "2020-01-01", "2020-12-31")
retMap3 = CcbCommonUtils.findTrades(d, "2019-01-01", "2019-12-31")
}
if(retMap1){
d.count1 = retMap1.count
d.avg_price1 = (int)retMap1.avg_price
}
if(retMap2){
d.count2 = retMap2.count
d.avg_price2 = (int)retMap2.avg_price
}
if(retMap3){
d.count3 = retMap3.count
d.avg_price3 = (int)retMap3.avg_price
}
if(!retMap1 || !retMap2 || !retMap3){
println d.city+" "+ d.region+" "+ d.name +"成交有一年为空过滤"
return null
}
def houses = EsSearchUtils.findSellHouseDetail(d.city, d.region, d.name)
def rooms = []
def areas = []
for(def h : houses){
rooms << h.room
areas << h.area
}
rooms = rooms.unique().sort()
areas = areas.unique().sort()
if(rooms){
d.room = rooms[0]+"-"+rooms[rooms.size()-1]
d.area = areas[0]+"-"+areas[areas.size()-1]
}
def zhidaoprice = priceMap.get(d.city+"#"+d.region)
def ret = [d.city, d.region, zhidaoprice, d.name, d.address, d.developer, d.cubage_percent, d.estate_company, d.household_count, d.house_year,d.room, d.area, d.count1, d.avg_price1,d.count2, d.avg_price2,d.count3, d.avg_price3]
println ret
return ret;
}
def standardRegion(filePath){
def city = null
def cityMap = new LinkedHashMap()
def index = 0
ExcelUtils.modify(filePath, { row ->
index ++
if(index >= 4){
if(row['C']){
city = row['C']
}
println city+" "+row['D']
def newRegion = null
if(city && row['D']){
newRegion = RegionData.formatRegion(city, row['D'])
row['F'] = newRegion
}
if(city && newRegion){
def regions = cityMap.get(city)
if(!regions){
regions = []
cityMap.put(city, regions)
}
if(!regions.contains(newRegion)){
regions << newRegion
}
}
}
});
return cityMap
}
def findCreditHightDistrict2(def city, def regions, def weekend, def priceMap, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
q.put("region", new BasicDBObject('$in', regions))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
if(regions.contains(d.region)){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice){
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def price = priceMap.get(d.city +"#"+d.region)
if(d.avg_price>price){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}else{
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}
return retMaps;
}
def getZhiDaoPrice(filePath){
def priceMap = [:]
def city = null
ExcelUtils.modify(filePath, { row ->
if(row['C']){
city = row['C']
}
if(city && row['F']){
def key = city +"#"+row['F']
if(!priceMap.containsKey(key)){
priceMap.put(key, row['E'])
}
}
})
return priceMap
}
public static def findShTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("fjCity", dist.city)
q.put("fjRegion", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("fjName", new BasicDBObject('$in', dist.alias))
}else{
q.put("fjName", dist.name)
}
q.put("contractTime", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = youda_esfhouse_coll.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.price){
total_price = total_price + d.price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
def findNearThirdPartyPrice(def city, regions){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", new BasicDBObject('$in', regions))
q.put("s_date", new BasicDBObject('$gte', 202105))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
}
@@ -24,21 +24,22 @@ class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) { static main(args) {
Date date=new Date(1622802609); // Date date=new Date(1622802609);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String str = format.format(date); // String str = format.format(date);
System.out.println(str); // System.out.println(str);
// def task = new LocationTest() def task = new LocationTest()
// task.run() // task.run()
// task.run2() // task.run2()
// task.run3() task.run3()
// task.run4() // task.run4()
} }
def run(){ def run(){
def regionPointsMap = findRegionRange() def city = "南昌"
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]); def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0 def index = 0
def rets = [] def rets = []
for(def dist : districts){ for(def dist : districts){
@@ -74,7 +75,7 @@ class LocationTest {
} }
} }
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内") ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内")
} }
def run2(){ def run2(){
@@ -133,10 +134,10 @@ class LocationTest {
def run3(){ def run3(){
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d-> ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d->
def distance = null def distance = null
if(d['G']){ if(d['H']){
distance = d['G'].toDouble() distance = d['H'].toDouble()
} }
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){ if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期") def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
@@ -180,10 +181,56 @@ class LocationTest {
udata.put("m_date", new Date()) udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
def dbo = DbUtils.seaweed.findOne(q) def dbo = DbUtils.seaweed.findOne(q)
d['K'] = "修正" d['K'] = "修正"
}
}else if(d['E'] && d['B'] != d['E'] && distance && distance<=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
if(mapName.contains("-") && mapName.endsWith("楼")){
mapName = mapName.substring(0, mapName.indexOf("-"))
}
if(d['C']==mapName){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['E'])
q.put("name", d['C'])
q.put("cat", 'district')
q.put("status", 0)
def dbo = DbUtils.seaweed.findOne(q)
if(!dbo){
q.put("region", d['B'])
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(d['D']=="高德"){
lng = (d['I'].toDouble()*10000000000).longValue()
lat = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
}else{
lng2 = (d['I'].toDouble()*10000000000).longValue()
lat2 = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng = (gmap.get("lng")*10000000000).longValue()
lat = (gmap.get("lat")*10000000000).longValue()
}
}
def udata = new BasicDBObject()
udata.put("region", d['E'])
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
d['K'] = "修正区域"
}
} }
} }
},0) },0)
@@ -366,13 +413,13 @@ class LocationTest {
return flag; return flag;
} }
def findRegionRange(){ def findRegionRange(city){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city","福州市") q.put("city",city+"市")
def dbcursor = POLYLINECOLL.find(q) def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:] def regionMap = [:]
for(def d : dbcursor){ for(def d : dbcursor){
def region = RegionData.formatRegion("福州",d.region) def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline) regionMap.put(region, d.polyline)
} }
return regionMap return regionMap
@@ -531,8 +531,10 @@ public class DataUtil {
if(!(address ==~ /.*(½»»ã|½»¿Ú|½»²æ¿Ú)$/)){ if(!(address ==~ /.*(½»»ã|½»¿Ú|½»²æ¿Ú)$/)){
address += "½»»ã" address += "½»»ã"
} }
}else{
address = null
} }
address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã") // address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
} }
return address return address
} }