+21
-13
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -40,12 +41,11 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists)
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rows.addAll(rets)
|
||||
println city +" "+rets.size()
|
||||
|
||||
@@ -57,10 +57,10 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
allDatas.addAll(rows)
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
|
||||
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
@@ -70,7 +70,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDatas, distCodeMap, dist)
|
||||
def ret = createData(cityCodeDataMap, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
@@ -87,10 +87,10 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(def cityCodeDatas, def distCodeMap, def dist){
|
||||
def createData(def cityCodeDataMap, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
try{
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << cityCodeDataMap.get(dist.city)
|
||||
if(dist.region){
|
||||
ret << distCodeMap.get(dist.region)
|
||||
}else{
|
||||
@@ -101,9 +101,17 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
|
||||
if(dist.lat2){
|
||||
def lat = dist.lat2/10000000000d
|
||||
def lng = dist.lng2/10000000000d
|
||||
def loactionMap = GeoUtils.bd09ToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
@@ -120,7 +128,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "物业小区"
|
||||
ret << "pa100"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().endsWith("弄")){
|
||||
|
||||
+45
-29
@@ -27,8 +27,8 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
@@ -44,35 +44,43 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
// def trades = getCityData(city)
|
||||
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
def trades = getCityData(city)
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
|
||||
def youDataTrade = getYouDaData()
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" 成交共:"+rets.size()
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData()
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
println city +" "+youdaDatas.size()
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
|
||||
|
||||
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rets)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
cityDatas.addAll(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false);
|
||||
def fileName = "/home/goojia/ccb/bigdata_deal.txt"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(rows)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
|
||||
// def rets = []
|
||||
// rets << headers
|
||||
// rets.addAll(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
@@ -114,9 +122,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
ret << trade.city
|
||||
if(trade.fjRegion){
|
||||
ret << distCodeMap.get(trade.fjRegion)
|
||||
}else{
|
||||
@@ -215,11 +223,11 @@ class BigdataDealBatch extends BaseBatch{
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
@@ -240,7 +248,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
@@ -248,9 +256,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
ret << trade.city
|
||||
if(trade.region){
|
||||
ret << distCodeMap.get(trade.region)
|
||||
}else{
|
||||
@@ -317,7 +325,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
@@ -373,15 +381,23 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
def startDate = null;
|
||||
if(citys.contains(city)){
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
|
||||
}else{
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
}
|
||||
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date()))
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate)
|
||||
if(trades){
|
||||
tradesList.addAll(trades)
|
||||
}
|
||||
|
||||
+82
-37
@@ -12,6 +12,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -54,7 +55,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch(), args);
|
||||
@@ -63,15 +65,19 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def index = 0
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
index++
|
||||
def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, distIds,20200701, 20210731);
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){
|
||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap,districts, distIds,startDate, endDate){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
@@ -85,36 +91,42 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeDatas.city_code
|
||||
def regionCode = distCodeMap.get(rMap.region)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = rMap.region
|
||||
m.region_code = regionCode
|
||||
m.district_id = rMap.get("_id").toString()
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
def commId = rMap.get("_id").toString()
|
||||
if(distIds.contains(commId)){
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret && ret.size()<100){
|
||||
ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||
}
|
||||
if(ret){
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def regionCode = distCodeMap.get(rMap.region)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = rMap.region
|
||||
m.region_code = regionCode
|
||||
m.district_id = commId
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
}
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -124,14 +136,27 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_forsale.txt"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt"
|
||||
}
|
||||
|
||||
println city + " 房源总量:"+retMap.size()
|
||||
// def allDatas = createSellDatas(city, retMap.values());
|
||||
def allDatas = retMap.values()
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true);
|
||||
def headers = []
|
||||
if(index == 1){
|
||||
headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
}
|
||||
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
|
||||
println "txt文件生成 "
|
||||
|
||||
// def rets = []
|
||||
// rets << headers
|
||||
// rets.addAll(allDatas)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -236,6 +261,15 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
@@ -358,8 +392,9 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
|
||||
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
@@ -518,4 +553,14 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
def getDistrictIds(def cityid){
|
||||
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
|
||||
|
||||
def distIds = new HashSet()
|
||||
sqlInstance.eachRow(sql,{it->
|
||||
distIds << it.commid
|
||||
});
|
||||
return distIds;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+46
-16
@@ -7,7 +7,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
|
||||
public static def citys = ["成都","沈阳","大连","北京", "上海","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","重庆","南京","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
|
||||
// public static def citys = ["武汉"]
|
||||
|
||||
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
|
||||
@@ -23,21 +23,10 @@ class CcbCommonUtils {
|
||||
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
public static def findCityCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def count = dbcursor.count()
|
||||
if(count >1){
|
||||
def data = null
|
||||
for(def db : dbcursor){
|
||||
if(db.city_code){
|
||||
data = db
|
||||
break
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
public static def findCityCodeValue(){
|
||||
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
|
||||
return cityCodeMap;
|
||||
}
|
||||
|
||||
public static def findAllDistrictCodeValue(city){
|
||||
@@ -105,4 +94,45 @@ class CcbCommonUtils {
|
||||
return lists
|
||||
}
|
||||
|
||||
|
||||
public static def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
|
||||
if(count && avg_price){
|
||||
def retMap = [:]
|
||||
retMap.count = count
|
||||
retMap.avg_price = avg_price
|
||||
return retMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
-4
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
results << headers
|
||||
results.addAll(rows)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8");
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\staging_district.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\staging_district.txt",headers, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
@@ -76,9 +76,9 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
def getFenQiMap(def dists){
|
||||
def fenQiMap = [:]
|
||||
for(def d : dists){
|
||||
if(d.master_district_name){
|
||||
// if(d.master_district_name){
|
||||
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
// }
|
||||
}
|
||||
return fenQiMap;
|
||||
}
|
||||
|
||||
+5
-3
@@ -19,7 +19,7 @@ import com.mongodb.DBCollection
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
@@ -30,7 +30,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]//CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
@@ -166,7 +166,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
map.put(db.region+"##"+db.name, db)
|
||||
if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
map.put(db.region+"##"+db.name, db)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
+22
@@ -698,6 +698,28 @@ class CommonDatautils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictByQuery(def q, def cat, def fields){
|
||||
|
||||
|
||||
+103
-35
@@ -2,6 +2,8 @@ package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DataUtil
|
||||
@@ -63,7 +65,7 @@ class AddThirdPriceDistrict {
|
||||
def resAddrMap = [:]
|
||||
def filterEstateTypeMap = [:]
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -82,14 +84,77 @@ class AddThirdPriceDistrict {
|
||||
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
|
||||
}
|
||||
def run(){
|
||||
exportDatas();
|
||||
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
// addInfo();
|
||||
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
|
||||
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
|
||||
// exportDatas();
|
||||
// getDataType("D:\\数据\\小区\\南昌第三方均价.xlsx");
|
||||
// addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx");
|
||||
// addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
// addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
}
|
||||
|
||||
def dataFilter(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def name = row['C']
|
||||
if(name){
|
||||
def flag = NameProcess.filterName(name)
|
||||
if(!flag){
|
||||
row['T'] = 1
|
||||
}else{
|
||||
if(name.toString().length()<3){
|
||||
if(row['N'] && row['N']>=10 && !row['J']){
|
||||
row['I'] = 25
|
||||
}else{
|
||||
row['I'] = 18
|
||||
}
|
||||
|
||||
}else{
|
||||
def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
|
||||
if(s1){
|
||||
row['I'] = 19
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(!name){
|
||||
row['I'] = 1
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
println "3.名称过滤完成"
|
||||
}
|
||||
|
||||
def addNewMapInfo(def path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['I']){
|
||||
|
||||
def gdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "高德")
|
||||
if(gdData){
|
||||
row['F'] = gdData.district
|
||||
row['G'] = gdData.name
|
||||
row['H'] = gdData.address
|
||||
row['I'] = gdData.type
|
||||
row['J'] = gdData.score
|
||||
row['K'] = gdData.lng
|
||||
row['L'] = gdData.lat
|
||||
}
|
||||
|
||||
def bdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "百度")
|
||||
if(bdData){
|
||||
row['M'] = bdData.district
|
||||
row['N'] = bdData.name
|
||||
row['O'] = bdData.address
|
||||
row['P'] = bdData.type
|
||||
row['Q'] = bdData.score
|
||||
row['R'] = bdData.lng
|
||||
row['S'] = bdData.lat
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def addCrawlMapInfo( def path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def flag = row['R']
|
||||
@@ -105,27 +170,27 @@ class AddThirdPriceDistrict {
|
||||
row['K'] = data.lat
|
||||
}
|
||||
|
||||
def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
|
||||
def bdRegion =null
|
||||
def bdName = null
|
||||
def bdAddress = null
|
||||
def bdLng = null
|
||||
def bdLat = null
|
||||
def bdEstateType = null
|
||||
if(baiduRets){
|
||||
bdRegion = baiduRets.region
|
||||
bdName = baiduRets.name
|
||||
bdAddress = baiduRets.address
|
||||
bdLng = baiduRets.lng
|
||||
bdLat = baiduRets.lat
|
||||
bdEstateType = baiduRets.estateType
|
||||
}
|
||||
row['L'] = bdRegion
|
||||
row['M'] = bdName
|
||||
row['N'] = bdAddress
|
||||
row['O'] = bdEstateType
|
||||
row['P'] = bdLng
|
||||
row['Q'] = bdLat
|
||||
// def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
|
||||
// def bdRegion =null
|
||||
// def bdName = null
|
||||
// def bdAddress = null
|
||||
// def bdLng = null
|
||||
// def bdLat = null
|
||||
// def bdEstateType = null
|
||||
// if(baiduRets){
|
||||
// bdRegion = baiduRets.region
|
||||
// bdName = baiduRets.name
|
||||
// bdAddress = baiduRets.address
|
||||
// bdLng = baiduRets.lng
|
||||
// bdLat = baiduRets.lat
|
||||
// bdEstateType = baiduRets.estateType
|
||||
// }
|
||||
// row['L'] = bdRegion
|
||||
// row['M'] = bdName
|
||||
// row['N'] = bdAddress
|
||||
// row['O'] = bdEstateType
|
||||
// row['P'] = bdLng
|
||||
// row['Q'] = bdLat
|
||||
}
|
||||
|
||||
},0);
|
||||
@@ -502,8 +567,8 @@ class AddThirdPriceDistrict {
|
||||
return thirdPartyColl.findOne(q)
|
||||
}
|
||||
|
||||
def addInfo(){
|
||||
|
||||
def addInfo(filePath){
|
||||
province = "江西"
|
||||
// def city = "苏州"
|
||||
// def datas = findMatchErrorDatas(city);
|
||||
// println "共【"+datas.size()+"】条"
|
||||
@@ -512,7 +577,7 @@ class AddThirdPriceDistrict {
|
||||
// def resMap = groupByName(datas);
|
||||
|
||||
def index = 0;
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx", { row ->
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def flag = row['R']
|
||||
def dgDatas = row['G']
|
||||
// if(!flag){
|
||||
@@ -956,11 +1021,11 @@ class AddThirdPriceDistrict {
|
||||
|
||||
def queryPoiMap(def city, def region, def name){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and adname = '" + region + "'")
|
||||
sqlBuffer.append(" and name = '" + name + "'")
|
||||
sqlBuffer.append(" and source = '高德'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
sqlBuffer.append(" and source = '高德' order by score() desc limit 1")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def map = [:]
|
||||
while (rs.next()) {
|
||||
@@ -969,8 +1034,11 @@ class AddThirdPriceDistrict {
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.type = rs.getString("type")
|
||||
map.lng = rs.getString("location.lon")
|
||||
map.lat = rs.getString("location.lat")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
+19
-14
@@ -64,24 +64,29 @@ class EsSearchUtils {
|
||||
sqlBuffer.append(" and region = '"+region+"'")
|
||||
sqlBuffer.append(" and district_name = '"+name+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" limit 100")
|
||||
sqlBuffer.append(" limit 200")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def rets = []
|
||||
while (rs.next()){
|
||||
def height = rs.getInt("height")
|
||||
def direction = rs.getString("direction")
|
||||
def fitment = rs.getString("fitment")
|
||||
def room = rs.getInt("room")
|
||||
def hall = rs.getInt("hall")
|
||||
def toilet = rs.getInt("toilet")
|
||||
def map = [:]
|
||||
map.height = height
|
||||
map.direction = direction
|
||||
map.fitment = fitment
|
||||
map.room = room
|
||||
map.hall = hall
|
||||
map.toilet = toilet
|
||||
rets << map
|
||||
if(room){
|
||||
def height = rs.getInt("height")
|
||||
def direction = rs.getString("direction")
|
||||
def fitment = rs.getString("fitment")
|
||||
def hall = rs.getInt("hall")
|
||||
def toilet = rs.getInt("toilet")
|
||||
def area = rs.getInt("area")
|
||||
def map = [:]
|
||||
map.height = height
|
||||
map.direction = direction
|
||||
map.fitment = fitment
|
||||
map.room = room
|
||||
map.hall = hall
|
||||
map.toilet = toilet
|
||||
map.area = area
|
||||
rets << map
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
@@ -50,14 +50,14 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
// def categorys = ["141202", "141203"]//中学,小学
|
||||
|
||||
// def prefix = "district_"
|
||||
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
def prefix = "district_"
|
||||
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
|
||||
// def prefix = "road_"
|
||||
// def categorys = ["190301"]//道路相关
|
||||
|
||||
def prefix = "village_"
|
||||
def categorys = ["190108","190109"]//村庄
|
||||
// def prefix = "village_"
|
||||
// def categorys = ["190108","190109"]//村庄
|
||||
|
||||
// def prefix = "company_"
|
||||
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
|
||||
@@ -119,7 +119,7 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys =["无锡"]
|
||||
def citys =["南昌"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
|
||||
@@ -28,8 +28,8 @@ class RegionPointCrawl {
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
def provinces = ["江西省"]
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
@@ -72,7 +72,7 @@ class RegionPointCrawl {
|
||||
for(def cityMap : cityList){
|
||||
if(cityMap.level == "city"){
|
||||
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
|
||||
saveGaoDeRegionCode(province, cityMap.districts);
|
||||
saveGaoDeRegionCode(province, cityMap.name, cityMap.districts);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,10 +81,10 @@ class RegionPointCrawl {
|
||||
}
|
||||
}
|
||||
|
||||
def saveGaoDeRegionCode(province, districts){
|
||||
def saveGaoDeRegionCode(province, cityName,districts){
|
||||
if(districts){
|
||||
for(def dist : districts){
|
||||
if(dist.level=="district"){
|
||||
if(dist.level=="district" || cityName=="东莞市"){
|
||||
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||
}
|
||||
}
|
||||
@@ -221,4 +221,3 @@ class RegionPointCrawl {
|
||||
return KeyList.get(count);
|
||||
}
|
||||
}
|
||||
H
|
||||
@@ -12,8 +12,10 @@ class FileUtils {
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
sb.append(headers.join("|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
if(headers){
|
||||
sb.append(headers.join("|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
}
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
+13
-1
@@ -15,7 +15,19 @@ class BestDistrictPriceExport {
|
||||
|
||||
static main(args) {
|
||||
def task = new BestDistrictPriceExport();
|
||||
task.run()
|
||||
//task.run()
|
||||
task.run2()
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def filePath = "d:\\数据\\广发\\楼盘.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def stat = findDistrictPrice(row['A'], row['B'], row['D'], "20210801")
|
||||
if(stat){
|
||||
row['S'] = stat.avg_price
|
||||
}
|
||||
println row['A']+" "+row['B']+" "+ row['C']+" "+row['S']
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+7
-10
@@ -24,10 +24,10 @@ class DataProcess {
|
||||
|
||||
static main(args) {
|
||||
//建行信用卡城市(临汾--晋城).xlsx
|
||||
def filePath = "D:\\数据\\建行银行\\建行信用卡入库小区V1.xlsx"
|
||||
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx"
|
||||
def task = new DataProcess();
|
||||
// task.batchSaveDataToDB(filePath)
|
||||
// task.mapBatchSaveDataToDB(filePath)
|
||||
task.mapBatchSaveDataToDB(filePath)
|
||||
// task.calcutePrice(filePath);
|
||||
// task.calcutePrice2();
|
||||
// task.deleteRepDist("株洲");
|
||||
@@ -35,7 +35,7 @@ class DataProcess {
|
||||
|
||||
|
||||
// task.standartName(filePath);
|
||||
task.saveGaoDeMap(filePath);
|
||||
// task.saveGaoDeMap(filePath);
|
||||
|
||||
// task.updateDBAddress();
|
||||
// task.updateRegion()
|
||||
@@ -120,13 +120,10 @@ class DataProcess {
|
||||
def lastDatas = []
|
||||
if(datas){
|
||||
def keys = []
|
||||
for(def data : datas){
|
||||
if(data['A']=="厦门" && data['I'] && data['I'].toString().startsWith("24")){
|
||||
for(def data : datas){//20,21,25,
|
||||
if(data['I'] && [26,27].contains(data['I'].toString().toDouble().toInteger())){
|
||||
def key = data['A']+"##"+data['B']+"##"+data['H']
|
||||
if(!keys.contains(key) && data['H'].size()>2){
|
||||
// if(data['I'].toInteger()==22){//&& data['N']<3
|
||||
// continue;
|
||||
// }
|
||||
if(!keys.contains(key) && data['H'] && data['H'].size()>2){
|
||||
keys << key
|
||||
lastDatas << data
|
||||
}
|
||||
@@ -168,7 +165,7 @@ class DataProcess {
|
||||
addIndex++
|
||||
def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null)
|
||||
println "${addIndex}【新增入库】"+ values
|
||||
// FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
|
||||
}else{
|
||||
println "【已存在】 ${d['A']} ${d['B']} ${d['H']}"
|
||||
}
|
||||
|
||||
+5
-3
@@ -5,6 +5,7 @@ import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
@@ -26,8 +27,9 @@ class ExportData {
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportData();
|
||||
// task.exportThird()
|
||||
task.exportMap();
|
||||
task.exportThird()
|
||||
// task.exportMap();
|
||||
|
||||
}
|
||||
|
||||
def exportMap(){
|
||||
@@ -72,7 +74,7 @@ class ExportData {
|
||||
|
||||
|
||||
def exportThird(){
|
||||
def citys = ["重庆"]
|
||||
def citys = ["南昌"]
|
||||
for(def city : citys){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","别名"]
|
||||
|
||||
+131
-22
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
@@ -7,6 +8,7 @@ import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
|
||||
|
||||
import java.util.regex.Pattern
|
||||
@@ -15,32 +17,39 @@ class MapDistrictProcess {
|
||||
static main(args) {
|
||||
def task = new MapDistrictProcess();
|
||||
task.run()
|
||||
// task.updateRegion()
|
||||
}
|
||||
|
||||
/**
|
||||
* 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路
|
||||
* 20:不同区域新增(库中无) 21:分期新增 22:分数在0.6以上
|
||||
* 20:不同区域新增(库中无) 21:分期新增 23:不同区域,距离相近 25:无匹配,可新增 22:有匹配,距离1000米以下,相似度在0.6以上 24:疑似匹配有误 27:人工判定入库
|
||||
* @return
|
||||
*/
|
||||
def run(){
|
||||
|
||||
def files = ["武汉.xlsx"]
|
||||
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
|
||||
def files = ["东莞.xlsx"]
|
||||
for(def fileName : files){
|
||||
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
|
||||
// typeFilter(filePath)
|
||||
// nameStandard(filePath);
|
||||
// dataFilter(filePath)
|
||||
matchDistDatas(filePath)
|
||||
typeFilter(filePath)
|
||||
nameStandard(filePath);
|
||||
dataFilter(filePath)
|
||||
houseCalculate(filePath);
|
||||
////// //统计是否存在多个区域
|
||||
matchDistDatas(filePath)
|
||||
moreRegionCalculate(filePath)
|
||||
|
||||
addFlag(filePath)
|
||||
mapMatch(filePath);
|
||||
baiduTypeFilter(filePath);
|
||||
baiduMap(filePath);
|
||||
// baiduMap(filePath);
|
||||
distanceFilter(filePath)
|
||||
predictType(filePath)
|
||||
noMatchFlag(filePath)
|
||||
matchFlag(filePath)
|
||||
addFeature(filePath)
|
||||
houseMatch(filePath);
|
||||
predictType(filePath)
|
||||
|
||||
// nameScore(filePath);
|
||||
}
|
||||
|
||||
@@ -55,7 +64,12 @@ class MapDistrictProcess {
|
||||
row['I'] = 1
|
||||
}else{
|
||||
if(name.toString().length()<3){
|
||||
row['I'] = 18
|
||||
if(row['N'] && row['N']>=10 && !row['J']){
|
||||
row['I'] = 25
|
||||
}else{
|
||||
row['I'] = 18
|
||||
}
|
||||
|
||||
}else{
|
||||
def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
|
||||
if(s1){
|
||||
@@ -70,7 +84,7 @@ class MapDistrictProcess {
|
||||
}
|
||||
|
||||
});
|
||||
println "2.名称过滤完成"
|
||||
println "3.名称过滤完成"
|
||||
}
|
||||
|
||||
def typeFilter(filePath){
|
||||
@@ -84,15 +98,30 @@ class MapDistrictProcess {
|
||||
println "1.类型过滤完成"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def baiduTypeFilter(filePath){
|
||||
def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"]
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def type = row['S']
|
||||
if(type && !types.contains(type)){
|
||||
if(type && row['Q'] && row['Q']==row['H'] && !types.contains(type)){
|
||||
row['I'] = 1
|
||||
}
|
||||
});
|
||||
println "1.类型过滤完成"
|
||||
println "9.百度类型过滤完成"
|
||||
}
|
||||
|
||||
def distanceFilter(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(!row['I']){
|
||||
if(row['X'] && row['Y']){
|
||||
if(row['X']<=500 && row['Y']>=0.5){
|
||||
row['I'] = 12
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
println "10.距离过滤完成"
|
||||
}
|
||||
|
||||
def nameStandard(filePath){
|
||||
@@ -105,12 +134,12 @@ class MapDistrictProcess {
|
||||
}
|
||||
|
||||
}, 0);
|
||||
println "1.名称标准化完成"
|
||||
println "2.名称标准化完成"
|
||||
}
|
||||
|
||||
def matchDistDatas(def path){
|
||||
def index = 0
|
||||
ExcelUtils.modify(path,{ List row ->
|
||||
ExcelUtils.multiThreadingModify(path,{ List row ->
|
||||
index ++
|
||||
if(!row['I'] && !row['J']){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
|
||||
@@ -126,13 +155,16 @@ class MapDistrictProcess {
|
||||
def maplat = baidu.y
|
||||
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
|
||||
if(score<0.3){
|
||||
score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['M'])
|
||||
}
|
||||
row['Y'] = score
|
||||
}
|
||||
}
|
||||
println index +" "+row
|
||||
}
|
||||
},0)
|
||||
println "3.小区匹配完成"
|
||||
println "5.小区匹配完成"
|
||||
}
|
||||
|
||||
def houseCalculate(filePath){
|
||||
@@ -141,6 +173,7 @@ class MapDistrictProcess {
|
||||
row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H'])
|
||||
}
|
||||
});
|
||||
println "4.房源统计完成"
|
||||
}
|
||||
|
||||
def moreRegionCalculate(filePath){
|
||||
@@ -161,8 +194,12 @@ class MapDistrictProcess {
|
||||
}
|
||||
}
|
||||
if(!regions.contains(row['B'])){
|
||||
row['O'] = regions.toString()
|
||||
row['I'] = 20
|
||||
if(row['X'] && row['X']<=500 && row['Y'] && row['Y']>=0.5){
|
||||
row['I'] = 23
|
||||
}else{
|
||||
row['O'] = regions.toString()
|
||||
row['I'] = 20
|
||||
}
|
||||
}else if(regions.contains(row['B'])){
|
||||
row['O'] = "已存在"
|
||||
row['I'] = 10
|
||||
@@ -170,6 +207,7 @@ class MapDistrictProcess {
|
||||
}
|
||||
}
|
||||
});
|
||||
println "6.区域过滤完成"
|
||||
}
|
||||
|
||||
def addFlag(def filePath){
|
||||
@@ -206,6 +244,8 @@ class MapDistrictProcess {
|
||||
println index + " " + row
|
||||
}
|
||||
});
|
||||
|
||||
println "7.打标记完成"
|
||||
}
|
||||
|
||||
def baiduMap(def filePath){
|
||||
@@ -233,6 +273,7 @@ class MapDistrictProcess {
|
||||
}
|
||||
}
|
||||
});
|
||||
println "11.特征过滤完成"
|
||||
}
|
||||
|
||||
def mapMatch(def filePath){
|
||||
@@ -260,16 +301,22 @@ class MapDistrictProcess {
|
||||
println index +" "+row
|
||||
},0);
|
||||
|
||||
println "4.地图匹配完成"
|
||||
println "8.地图匹配完成"
|
||||
}
|
||||
|
||||
def houseMatch(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(!row['I']) {
|
||||
def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
|
||||
if(rMap){
|
||||
row['I'] = rMap.flag
|
||||
}
|
||||
// def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
|
||||
// if(rMap){
|
||||
// row['I'] = rMap.flag
|
||||
// }
|
||||
|
||||
if(row['J']){
|
||||
if(row['X'] && row['Y'] ){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
@@ -314,6 +361,41 @@ class MapDistrictProcess {
|
||||
},0);
|
||||
}
|
||||
|
||||
def noMatchFlag(filePath) {
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if (!row['I'] && !row['J']) {
|
||||
if(row['N']>0){
|
||||
row['I'] = 25
|
||||
}else{
|
||||
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
|
||||
row['I'] = 25
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def matchFlag(filePath) {
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if (!row['I'] && row['J'] && row['X'] && row['Y']) {
|
||||
if(row['X']<=1000 && row['Y']>=0.6){
|
||||
row['I'] = 22
|
||||
}else if(row['X']<=600 && row['Y']>=0.5){
|
||||
row['I'] = 22
|
||||
}else if(row['X']<=1300 && row['Y']>=0.8){
|
||||
row['I'] = 22
|
||||
}else{
|
||||
row['I'] = 24
|
||||
}
|
||||
}else if(!row['I'] && row['J'] && row['Y']==0){
|
||||
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
|
||||
row['I'] = 26
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def nameScore(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(!row['I']) {
|
||||
@@ -343,4 +425,31 @@ class MapDistrictProcess {
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def updateRegion(){
|
||||
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx";
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def region = getRegionByLocation(row['F'], row['G']);
|
||||
if(region){
|
||||
row['B'] = RegionData.formatRegion(row['A'], region)
|
||||
}
|
||||
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def getRegionByLocation(lng, lat){
|
||||
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
|
||||
|
||||
if(jsonObject && jsonObject.info=="OK"){
|
||||
def compent = jsonObject.regeocode.addressComponent
|
||||
if(compent){
|
||||
def township = compent.township
|
||||
return township;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+13
-11
@@ -7,7 +7,9 @@ import java.util.regex.Pattern
|
||||
class NameProcess {
|
||||
|
||||
static def filterName(def name){
|
||||
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心","养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会"]
|
||||
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心",
|
||||
"养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房",
|
||||
"合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼"]
|
||||
def isAdd = true;
|
||||
for(def key : keys){
|
||||
if(name.contains(key)){
|
||||
@@ -17,7 +19,7 @@ class NameProcess {
|
||||
}
|
||||
|
||||
if(isAdd){
|
||||
def endKeys = ["路","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍"]
|
||||
def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门"]
|
||||
for(def key : endKeys){
|
||||
if(name.toString().endsWith(key)){
|
||||
isAdd = false;
|
||||
@@ -27,7 +29,7 @@ class NameProcess {
|
||||
}
|
||||
|
||||
if(isAdd){
|
||||
def startKeys = ["FQ"]
|
||||
def startKeys = ["FQ","社区"]
|
||||
for(def key : startKeys){
|
||||
if(name.toString().endsWith(key)||name.toString().startsWith(key)){
|
||||
isAdd = false;
|
||||
@@ -36,7 +38,7 @@ class NameProcess {
|
||||
}
|
||||
}
|
||||
if(isAdd){
|
||||
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房"]
|
||||
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房", "家属宿舍楼","卫生生活区"]
|
||||
for(def str : strs){
|
||||
if(str == name){
|
||||
isAdd = false;
|
||||
@@ -56,7 +58,7 @@ class NameProcess {
|
||||
}
|
||||
}
|
||||
//.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号")
|
||||
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
|
||||
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
|
||||
name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","")
|
||||
if(name && name.contains("区")){
|
||||
def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区"))
|
||||
@@ -191,12 +193,12 @@ class NameProcess {
|
||||
if(str == name){
|
||||
return null
|
||||
}
|
||||
if(name){
|
||||
s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
|
||||
if(s1){
|
||||
return null
|
||||
}
|
||||
}
|
||||
// if(name){
|
||||
// s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
|
||||
// if(s1){
|
||||
// return null
|
||||
// }
|
||||
// }
|
||||
|
||||
return name.toUpperCase()
|
||||
}
|
||||
|
||||
+335
@@ -0,0 +1,335 @@
|
||||
package com.fangjia.script.gj.guangfa
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class GuangfaBlankSelectDistrict {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new GuangfaBlankSelectDistrict();
|
||||
task.run()
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def file = "D:\\数据\\广发\\获取楼盘信息需求(发送版).xlsx"
|
||||
def cityMap = standardRegion(file)
|
||||
def priceMap = getZhiDaoPrice(file)
|
||||
selectDistricts(cityMap, priceMap)
|
||||
println "写入完成"
|
||||
}
|
||||
|
||||
def selectDistricts(cityMap, priceMap){
|
||||
def rets = []
|
||||
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
|
||||
cityMap.each{city, regions->
|
||||
if(["西安","东莞","杭州"].contains(city)){
|
||||
def thirdPriceMap = findNearThirdPartyPrice(city, regions)
|
||||
def retMaps = findCreditHightDistrict2(city, regions, "2021-08-01", priceMap, thirdPriceMap)
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
|
||||
//基本情况合格的小区
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
if(regions.contains(d.region)){// && d.household_count && d.household_count>=50
|
||||
def distKeys = retMaps.get(d.region)
|
||||
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
|
||||
if(d.finish_building_date){
|
||||
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
|
||||
if(year){
|
||||
def mod = 2021-year
|
||||
if(mod <=15){
|
||||
d.house_year = mod
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
// if(d.finish_building_date){
|
||||
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
|
||||
// if(year){
|
||||
// def mod = 2021-year
|
||||
// if(mod <=15){
|
||||
// def distKeys = retMaps.get(d.region)
|
||||
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
|
||||
// d.house_year = mod
|
||||
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
println city +" 基本情况合格的小区:"+distMap.size()
|
||||
|
||||
for(def r : regions){
|
||||
def names = retMaps.get(r)
|
||||
if(names){
|
||||
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
|
||||
rets.addAll(results)
|
||||
}
|
||||
}
|
||||
println "结束"
|
||||
}
|
||||
}
|
||||
|
||||
println "开始写入Excel....."+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(names.size())
|
||||
|
||||
for(def i=0;i<names.size();i++){
|
||||
def key = names[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def d = distMap.get(key)
|
||||
if(d){
|
||||
def ret = getDetailInfo(d, priceMap)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
// latch.await()
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
return rets
|
||||
}
|
||||
|
||||
def getDetailInfo(d, priceMap){
|
||||
def retMap1 = null;
|
||||
def retMap2 = null;
|
||||
def retMap3 = null
|
||||
if(d.city == "上海"){
|
||||
retMap1 = findShTrades(d, "2021-01-01", "2021-07-01")
|
||||
retMap2 = findShTrades(d, "2020-01-01", "2020-12-31")
|
||||
retMap3 = findShTrades(d, "2019-01-01", "2019-12-31")
|
||||
}else{
|
||||
retMap1 = CcbCommonUtils.findTrades(d, "2021-01-01", "2021-07-01")
|
||||
retMap2 = CcbCommonUtils.findTrades(d, "2020-01-01", "2020-12-31")
|
||||
retMap3 = CcbCommonUtils.findTrades(d, "2019-01-01", "2019-12-31")
|
||||
}
|
||||
if(retMap1){
|
||||
d.count1 = retMap1.count
|
||||
d.avg_price1 = (int)retMap1.avg_price
|
||||
}
|
||||
if(retMap2){
|
||||
d.count2 = retMap2.count
|
||||
d.avg_price2 = (int)retMap2.avg_price
|
||||
}
|
||||
if(retMap3){
|
||||
d.count3 = retMap3.count
|
||||
d.avg_price3 = (int)retMap3.avg_price
|
||||
}
|
||||
if(!retMap1 || !retMap2 || !retMap3){
|
||||
println d.city+" "+ d.region+" "+ d.name +"成交有一年为空过滤"
|
||||
return null
|
||||
}
|
||||
|
||||
def houses = EsSearchUtils.findSellHouseDetail(d.city, d.region, d.name)
|
||||
def rooms = []
|
||||
def areas = []
|
||||
for(def h : houses){
|
||||
rooms << h.room
|
||||
areas << h.area
|
||||
}
|
||||
rooms = rooms.unique().sort()
|
||||
areas = areas.unique().sort()
|
||||
if(rooms){
|
||||
d.room = rooms[0]+"-"+rooms[rooms.size()-1]
|
||||
d.area = areas[0]+"-"+areas[areas.size()-1]
|
||||
}
|
||||
|
||||
def zhidaoprice = priceMap.get(d.city+"#"+d.region)
|
||||
def ret = [d.city, d.region, zhidaoprice, d.name, d.address, d.developer, d.cubage_percent, d.estate_company, d.household_count, d.house_year,d.room, d.area, d.count1, d.avg_price1,d.count2, d.avg_price2,d.count3, d.avg_price3]
|
||||
println ret
|
||||
return ret;
|
||||
}
|
||||
|
||||
def standardRegion(filePath){
|
||||
def city = null
|
||||
def cityMap = new LinkedHashMap()
|
||||
def index = 0
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
index ++
|
||||
if(index >= 4){
|
||||
if(row['C']){
|
||||
city = row['C']
|
||||
}
|
||||
println city+" "+row['D']
|
||||
def newRegion = null
|
||||
if(city && row['D']){
|
||||
newRegion = RegionData.formatRegion(city, row['D'])
|
||||
row['F'] = newRegion
|
||||
}
|
||||
if(city && newRegion){
|
||||
def regions = cityMap.get(city)
|
||||
if(!regions){
|
||||
regions = []
|
||||
cityMap.put(city, regions)
|
||||
}
|
||||
if(!regions.contains(newRegion)){
|
||||
regions << newRegion
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return cityMap
|
||||
}
|
||||
|
||||
|
||||
def findCreditHightDistrict2(def city, def regions, def weekend, def priceMap, def thirdPriceMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
if(regions.contains(d.region)){
|
||||
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
|
||||
if(thirdPrice){
|
||||
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
|
||||
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
|
||||
def price = priceMap.get(d.city +"#"+d.region)
|
||||
if(d.avg_price>price){
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
rets << d.city+"##"+d.region+"##"+d.district_name
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
rets << d.city+"##"+d.region+"##"+d.district_name
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return retMaps;
|
||||
}
|
||||
|
||||
def getZhiDaoPrice(filePath){
|
||||
def priceMap = [:]
|
||||
def city = null
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if(row['C']){
|
||||
city = row['C']
|
||||
}
|
||||
if(city && row['F']){
|
||||
def key = city +"#"+row['F']
|
||||
if(!priceMap.containsKey(key)){
|
||||
priceMap.put(key, row['E'])
|
||||
}
|
||||
}
|
||||
})
|
||||
return priceMap
|
||||
}
|
||||
|
||||
public static def findShTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fjCity", dist.city)
|
||||
q.put("fjRegion", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("fjName", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("fjName", dist.name)
|
||||
}
|
||||
|
||||
q.put("contractTime", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = youda_esfhouse_coll.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.price){
|
||||
total_price = total_price + d.price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
|
||||
if(count && avg_price){
|
||||
def retMap = [:]
|
||||
retMap.count = count
|
||||
retMap.avg_price = avg_price
|
||||
return retMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findNearThirdPartyPrice(def city, regions){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
q.put("s_date", new BasicDBObject('$gte', 202105))
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def thirdPriceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.region+"#"+d.name
|
||||
if(!thirdPriceMap.containsKey(key)){
|
||||
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return thirdPriceMap
|
||||
}
|
||||
}
|
||||
+63
-16
@@ -24,21 +24,22 @@ class LocationTest {
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
Date date=new Date(1622802609);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
String str = format.format(date);
|
||||
System.out.println(str);
|
||||
// Date date=new Date(1622802609);
|
||||
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
// String str = format.format(date);
|
||||
// System.out.println(str);
|
||||
|
||||
// def task = new LocationTest()
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
task.run3()
|
||||
// task.run4()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def regionPointsMap = findRegionRange()
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def city = "南昌"
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def index = 0
|
||||
def rets = []
|
||||
for(def dist : districts){
|
||||
@@ -74,7 +75,7 @@ class LocationTest {
|
||||
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内")
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
@@ -133,10 +134,10 @@ class LocationTest {
|
||||
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d->
|
||||
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d->
|
||||
def distance = null
|
||||
if(d['G']){
|
||||
distance = d['G'].toDouble()
|
||||
if(d['H']){
|
||||
distance = d['H'].toDouble()
|
||||
}
|
||||
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
|
||||
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
|
||||
@@ -180,10 +181,56 @@ class LocationTest {
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
|
||||
def dbo = DbUtils.seaweed.findOne(q)
|
||||
d['K'] = "修正"
|
||||
}
|
||||
}else if(d['E'] && d['B'] != d['E'] && distance && distance<=1000){
|
||||
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
|
||||
if(mapName.contains("-") && mapName.endsWith("楼")){
|
||||
mapName = mapName.substring(0, mapName.indexOf("-"))
|
||||
}
|
||||
if(d['C']==mapName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['E'])
|
||||
q.put("name", d['C'])
|
||||
q.put("cat", 'district')
|
||||
q.put("status", 0)
|
||||
def dbo = DbUtils.seaweed.findOne(q)
|
||||
if(!dbo){
|
||||
q.put("region", d['B'])
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(d['D']=="高德"){
|
||||
lng = (d['I'].toDouble()*10000000000).longValue()
|
||||
lat = (d['J'].toDouble()*10000000000).longValue()
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
|
||||
if(gmap){
|
||||
lng2 = (gmap.get("lng")*10000000000).longValue()
|
||||
lat2 = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
|
||||
}else{
|
||||
lng2 = (d['I'].toDouble()*10000000000).longValue()
|
||||
lat2 = (d['J'].toDouble()*10000000000).longValue()
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
|
||||
if(gmap){
|
||||
lng = (gmap.get("lng")*10000000000).longValue()
|
||||
lat = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", d['E'])
|
||||
udata.put("lng", lng)
|
||||
udata.put("lat", lat)
|
||||
udata.put("lng2", lng2)
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
d['K'] = "修正区域"
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
@@ -366,13 +413,13 @@ class LocationTest {
|
||||
return flag;
|
||||
}
|
||||
|
||||
def findRegionRange(){
|
||||
def findRegionRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州市")
|
||||
q.put("city",city+"市")
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion("福州",d.region)
|
||||
def region = RegionData.formatRegion(city,d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
|
||||
@@ -531,8 +531,10 @@ public class DataUtil {
|
||||
if(!(address ==~ /.*(½»»ã|½»¿Ú|½»²æ¿Ú)$/)){
|
||||
address += "½»»ã"
|
||||
}
|
||||
}else{
|
||||
address = null
|
||||
}
|
||||
address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
|
||||
// address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
|
||||
}
|
||||
return address
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user