Merge remote-tracking branch 'origin/script_gj'
This commit is contained in:
+3
-3
@@ -3,13 +3,14 @@ package com.fangjia.script.ana
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
// 分期分区与主小区映射关系表
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class FindStageDistrictBatch extends BaseBatch {
|
||||
|
||||
@@ -24,7 +25,7 @@ class FindStageDistrictBatch extends BaseBatch {
|
||||
@Override
|
||||
void process(Object args) {
|
||||
CommonUtils.findAllCity().each { def city ->
|
||||
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
|
||||
CommonDatautils.SEAWEED_COLL.find(DbUtils.buildSeaweedQuery2(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
|
||||
if (dbo.name =~ STAGE_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
|
||||
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
|
||||
@@ -40,7 +41,6 @@ class FindStageDistrictBatch extends BaseBatch {
|
||||
println(dbo)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
@@ -21,6 +21,7 @@ import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 通知计算二手房均价
|
||||
* 发送平米租金命令
|
||||
* @author duxiaoyu
|
||||
*
|
||||
*/
|
||||
|
||||
+17
-11
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -72,21 +72,27 @@ for(def d : datas){
|
||||
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
def trade_date = d['J']
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
map.area = d['F'].replace("平","").toString().toDouble()
|
||||
map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble())
|
||||
if(d['I']){
|
||||
map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000)
|
||||
}
|
||||
|
||||
map.trade_date = d['J']
|
||||
// def trade_date = d['J']
|
||||
// if(trade_date){
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||
// }
|
||||
// else{
|
||||
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
// }
|
||||
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
println map
|
||||
DEAL_PRICE_COLL.save(map)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ class AddThirdPartyPrice {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def c : citys){
|
||||
println c +" "+cityCodeDatas.get(c)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c)
|
||||
def rets = createDate(dists, distCodeMap)
|
||||
println dists.size() +" "+rets.size()
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
|
||||
*/
|
||||
class AddThirdPriceSample {
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceSample()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
for(def i=1;i<19;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
|
||||
if(d['A'] && d['C']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
|
||||
if(dist){
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
|
||||
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
|
||||
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
|
||||
}else{
|
||||
println d['A'] +" 没查到"
|
||||
}
|
||||
}
|
||||
}, i)
|
||||
|
||||
println(i)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+10
-9
@@ -24,15 +24,16 @@ class BatchAddPersonGuiPrice {
|
||||
|
||||
def run(){
|
||||
|
||||
def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx"
|
||||
def path = "D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx"
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
for(def d : datas){
|
||||
def row = findGuidingPrice(d['B'], d['C'], d['D'])
|
||||
def row = findGuidingPrice(d['B'], d['C'], d['C'])
|
||||
if(!row){
|
||||
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
|
||||
// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
save(d['A'], d['B'], d['C'], d['Q'], obj.estate_type2, d['P'])
|
||||
//发重算均价命令
|
||||
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date())
|
||||
addPrice(d['A'], d['B'], d['C'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),new Date())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,13 +44,13 @@ class BatchAddPersonGuiPrice {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("guiding_person" , "付贵生")
|
||||
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd")))
|
||||
// q.put("guiding_person" , person)
|
||||
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-03-20", "yyyy-MM-dd")))
|
||||
def dbo = GUIDING_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def save(city, region, name, price, estate_type){
|
||||
def save(city, region, name, price, estate_type, person){
|
||||
DBObject obj = new BasicDBObject();
|
||||
obj.put("city", city);
|
||||
obj.put("region", region);
|
||||
@@ -62,7 +63,7 @@ class BatchAddPersonGuiPrice {
|
||||
obj.put("m_date", new Date());
|
||||
obj.put("estate_type" , estate_type);
|
||||
obj.put("house_type" , 0)
|
||||
obj.put("guiding_person" , "陈杰")
|
||||
obj.put("guiding_person" , person)
|
||||
GUIDING_PRICE_COLL.save(obj);
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
def alias = d.alias;
|
||||
|
||||
+13
-10
@@ -34,13 +34,15 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def fangjiaDists = new HashSet()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
@@ -52,14 +54,18 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
|
||||
def allDists = CcbCommonUtils.findAllDistrictByCity(null, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
for(def d : allDists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
for(def city : citys){
|
||||
|
||||
index ++
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
|
||||
println "加载建行小区量"
|
||||
|
||||
def ccbMaps = null
|
||||
if(today<=10){
|
||||
@@ -67,11 +73,11 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}else{
|
||||
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
}
|
||||
|
||||
println "组装新数据"
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()+" 结果:"+rows.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
@@ -103,10 +109,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
|
||||
+54
-13
@@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -34,7 +35,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
@@ -61,13 +62,21 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
// def objectId = dateToObjectId(date)
|
||||
// println objectId
|
||||
|
||||
//664a22000000000000000000 5-20
|
||||
//664b73800000000000000000 5-21
|
||||
def citys =CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
@@ -180,7 +189,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
latch.await();
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -376,7 +385,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
def data = distCodeMap.get(trade.match_region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
@@ -529,7 +538,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
@@ -679,6 +688,16 @@ class BigdataDealBatch extends BaseBatch{
|
||||
return map
|
||||
}
|
||||
|
||||
def updateCrawlTradeDate(){
|
||||
def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20"
|
||||
def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2)
|
||||
def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
println(objectId)
|
||||
CcbCommonUtils.updateTrades(startDate, endDate, objectId)
|
||||
}
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
@@ -701,22 +720,33 @@ class BigdataDealBatch extends BaseBatch{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
if(region || (!region && ["澄迈"].contains(t.city))) {
|
||||
if(t.house_use && t.house_use.contains("商住")){
|
||||
if(!t.name.contains("商住")){
|
||||
t.name = t.name +"商住楼"
|
||||
}
|
||||
}
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
def flag = false
|
||||
def address = null
|
||||
if(t.address){
|
||||
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
address = t.address
|
||||
}
|
||||
}
|
||||
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
|
||||
if(!rMap && !t.name.toString().endsWith("小区")){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
|
||||
}
|
||||
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else if (rMap && ["澄迈"].contains(rMap.city)){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else{
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
}
|
||||
@@ -733,7 +763,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
)
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
@@ -816,4 +846,15 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def dateToObjectId(Date date) {
|
||||
// 获取时间戳(单位:秒)
|
||||
def timestamp = (date.time / 1000) as int
|
||||
def hexTimestamp = Integer.toHexString(timestamp)
|
||||
|
||||
// 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值
|
||||
def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000'
|
||||
return new ObjectId(objectIdStr)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
//测试通过匹配接口配房源效果
|
||||
|
||||
+60
-17
@@ -7,17 +7,22 @@ import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
//
|
||||
// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
|
||||
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
|
||||
public static def citys_fjw_all = ["长春","上海","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
|
||||
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
||||
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
|
||||
|
||||
@@ -69,6 +74,10 @@ class CcbCommonUtils {
|
||||
citys = CcbCommonUtils.citys_252
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="all"){
|
||||
citys = CcbCommonUtils.citys_fjw_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}
|
||||
}else{
|
||||
citys = CcbCommonUtils.citys_all
|
||||
@@ -228,7 +237,7 @@ class CcbCommonUtils {
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
@@ -428,16 +437,19 @@ class CcbCommonUtils {
|
||||
return rows
|
||||
}
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields, def kind){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
if(city){
|
||||
q.put("city", city);
|
||||
}
|
||||
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","汤堡小区")
|
||||
// q.put("name","一米阳光一期")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -445,14 +457,13 @@ class CcbCommonUtils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
def rets = new HashSet()
|
||||
def ids = new HashSet()
|
||||
|
||||
def ids = []
|
||||
for(def d : dbsour){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(kind){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
@@ -460,6 +471,16 @@ class CcbCommonUtils {
|
||||
ids << d._id.toString()
|
||||
rets << d
|
||||
}
|
||||
}else{
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
if(!ids.contains(d._id.toString())){
|
||||
ids << d._id.toString()
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println "[${city}] 小区量:"+rets.size()
|
||||
@@ -620,7 +641,8 @@ class CcbCommonUtils {
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', false))
|
||||
// q.put("district_name","中海寰宇天下")
|
||||
// q.put("district_name","新明胡同")
|
||||
// q.put("address","桂苑大道135号")
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
@@ -665,15 +687,22 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
public static def deleteGuidedPrice(def dealdate){
|
||||
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
|
||||
|
||||
def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'"
|
||||
pgsqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findGuiDingIdsByCity(){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
pgsqlInstance.eachRow("SELECT gujia_fsid FROM base.t0831_guided_price_gj_h",{it->
|
||||
filter.put(it.gujia_fsid)
|
||||
});
|
||||
return filter
|
||||
}
|
||||
|
||||
|
||||
public static def findDistGovPrice(def city){
|
||||
def ids = []
|
||||
pgsqlInstance.eachRow("SELECT gujia_bds_id from base.t0831_community_guide_price_gj_h where urbn_nm='${city}'",{it->
|
||||
@@ -702,4 +731,18 @@ class CcbCommonUtils {
|
||||
return map
|
||||
}
|
||||
|
||||
public static def updateTrades(startDate, endDate, cDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("_id", new BasicDBObject('$gte', cDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
if(count>0){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("trade_date", endDate)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("flag", "补漏变更成交时间")
|
||||
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+27
-8
@@ -387,26 +387,45 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
||||
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
||||
def total = 0
|
||||
def cityid = cityCodeDatas.get(city)
|
||||
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
set.add(city)
|
||||
}
|
||||
for(def m : rets){
|
||||
def cityId = cityCodeDatas.get(city)
|
||||
|
||||
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
// sqlInstance236.executeUpdate(sql)
|
||||
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
def total = 0
|
||||
pgsqlInstance.eachRow(querySql, { r ->
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println city +" "+m.commid+" (按价格)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}else{
|
||||
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityId}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
|
||||
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.eachRow(querySql, { r ->
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println city +" "+m.commid+" (按小区)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println set.toString()
|
||||
@@ -432,8 +451,8 @@ class DealPriceDataProcess {
|
||||
return rets
|
||||
}
|
||||
|
||||
def findPeGteThirdParty2(cityName, yearmonth){
|
||||
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>20"
|
||||
def findPeGteThirdParty2(cityid, yearmonth){
|
||||
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
|
||||
def rets = []
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
|
||||
+19
-5
@@ -47,16 +47,22 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def rows = []
|
||||
|
||||
CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
|
||||
def priceIdsfilter = CcbCommonUtils.findGuiDingIdsByCity()
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-27")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
//近3个月人工指导价
|
||||
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
|
||||
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
@@ -80,7 +86,8 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
println "人工指导价总计:"+rows.size()
|
||||
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
@@ -142,11 +149,18 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
|
||||
if(priceIdsfilter.mightContain(ps._id.toString())){
|
||||
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
|
||||
continue
|
||||
}
|
||||
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
@@ -207,7 +221,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
|
||||
+15
@@ -2,6 +2,9 @@ package com.fangjia.script.ccbevaluation
|
||||
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* ½¨ÐÐʹÓÃ
|
||||
*/
|
||||
class PgsqlCommUtils {
|
||||
|
||||
public static def database = "warehouse"
|
||||
@@ -19,6 +22,9 @@ class PgsqlCommUtils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static def findAllDistrictCount(def cityid){
|
||||
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
|
||||
def count = 0
|
||||
@@ -27,4 +33,13 @@ class PgsqlCommUtils {
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
public static def findDistrictMap(){
|
||||
def sql = "select urbn_id,gujia_admndiv_id,gujia_admndiv_nm from base.t0831_bigdata_district_gj_h"
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
map.put(r.urbn_id+"#"+r.gujia_admndiv_id, r.gujia_admndiv_nm)
|
||||
});
|
||||
return map
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null)
|
||||
def fenQiMap = getFenQiMap(dists);
|
||||
def rets = createCityData(dists, fenQiMap)
|
||||
rows.addAll(rets)
|
||||
|
||||
+194
-25
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
@@ -198,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
|
||||
}
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
|
||||
// if(d['G']){
|
||||
// def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
// if(obj){
|
||||
// d['H'] = obj.name
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
////def url = "https://sh.anjuke.com"
|
||||
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
@@ -601,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
|
||||
////},0)
|
||||
//
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def dbo = CommonDatautils.findDistrictById(d['D'])
|
||||
// def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dbo){
|
||||
// def udata = new BasicDBObject()
|
||||
//// udata.put("name_cat", 11)
|
||||
@@ -619,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
|
||||
if(d['A'] && d['B']){
|
||||
def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dbo){
|
||||
def alias = null
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = dbo.alias
|
||||
}
|
||||
if(!alias.contains(d['B'])){
|
||||
alias << d['B']
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dbo._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+"别名已存在"
|
||||
}
|
||||
|
||||
}else{
|
||||
println dbo._id.toString() +" 没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||
//for(def c : citys){
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
|
||||
@@ -1288,35 +1331,158 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['J']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['L'] = dist.status-dist.name_cat
|
||||
//// }
|
||||
//// }
|
||||
//// if(d['J'] instanceof String){
|
||||
// def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
// if(dist){
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist._id.toString()
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
//
|
||||
//// d['H'] = dist.name
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(file, { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['C'] = dist.city
|
||||
// d['D'] = dist.region
|
||||
// d['E'] = dist.name
|
||||
// def random = RandomUtil.random(-300,50)
|
||||
// d['F'] = random + d['B']
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
// if(d['B']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(obj){
|
||||
// def markMap = obj.mark_map
|
||||
// if(!obj.mark_map){
|
||||
// markMap = [:]
|
||||
// }
|
||||
// def data = new BasicDBObject()
|
||||
// markMap.put("exclusive", "是")
|
||||
// data.put("mark_map", markMap)
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
//
|
||||
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
|
||||
//ExcelUtils.modify(filePath, { d ->
|
||||
// if(d['A']){
|
||||
//// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
|
||||
// if(list){
|
||||
// if(list.size()==1){
|
||||
// def dbo = list[0]
|
||||
// if(dbo){
|
||||
// def data = new BasicDBObject()
|
||||
// data.put("region", d['B'])
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }else{
|
||||
// println d['C'] +" 有多个"
|
||||
// }
|
||||
// }
|
||||
if(!d['I']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(dist){
|
||||
d['F'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
d['G'] = dist.status
|
||||
// if(d['A']=="天门" || d['A']=="迪庆"){
|
||||
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
|
||||
// d['K'] = val.toInteger()
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
//// if(obj){
|
||||
//// def lng = d['G']
|
||||
//// def lat =d['H']
|
||||
////
|
||||
//// // 高德转百度
|
||||
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
//// def lng2 = baidu.x
|
||||
//// def lat2 = baidu.y
|
||||
////
|
||||
////
|
||||
//// def data = new BasicDBObject()
|
||||
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
|
||||
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
|
||||
//// data.put("lng", lng_new)
|
||||
//// data.put("lat", lat_new)
|
||||
////
|
||||
//// Long dLat2 = lat2 * 10000000000L
|
||||
//// Long dLng2 = lng2 * 10000000000L
|
||||
//// data.put("lat2", dLat2)
|
||||
//// data.put("lng2", dLng2)
|
||||
////
|
||||
////// data.put("status", 1)
|
||||
//// data.put("m_date", new Date())
|
||||
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
|
||||
//// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// if(d['B'] && !d['D']){
|
||||
// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
|
||||
// if(jsonObject != null && jsonObject.credit>0) {
|
||||
// d['D'] = jsonObject.get("city")
|
||||
// d['E'] = jsonObject.get("district")
|
||||
// d['F'] = jsonObject.get("name")
|
||||
// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||
// if(dbo){
|
||||
// d['G'] = dbo.status
|
||||
// d['H'] = dbo.name_cat
|
||||
// d['I'] = dbo.estate_type2
|
||||
// d['J'] = "接口匹配"
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
|
||||
//// if(!objs){
|
||||
//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
|
||||
//// }
|
||||
//// if(objs && objs.size()==1){
|
||||
//// def o = objs[0]
|
||||
//// if(o){
|
||||
//// d['D'] = o.city
|
||||
//// d['E'] = o.region
|
||||
//// d['F'] = o.name
|
||||
//// d['G'] = o.status
|
||||
//// d['H'] = o.name_cat
|
||||
//// d['I'] = o.estate_type2
|
||||
//// d['J'] = "别名"
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//
|
||||
// }
|
||||
//})
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
//// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
@@ -1379,6 +1545,9 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
//citys = CcbCommonUtils.citys_all
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
|
||||
if(d['G'] && d['G'] instanceof String){
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
if(obj){
|
||||
d['H'] = obj.name
|
||||
}
|
||||
}
|
||||
})
|
||||
+22
-11
@@ -47,14 +47,6 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
// def gocPrices = CcbCommonUtils.findDistGovPrice("深圳")
|
||||
// def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val=202303 and gujia_bds_id=?"
|
||||
// pgsqlInstance.withBatch(500, sql){ ps->
|
||||
// for(def id : gocPrices){
|
||||
// ps.addBatch(id)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
def index = 0
|
||||
@@ -71,7 +63,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
|
||||
|
||||
@@ -265,6 +257,12 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
if(third.get("csjj_esf_price")){
|
||||
price_map.csjj_avg_price = third.get("csjj_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -311,6 +309,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "房产超市"
|
||||
}else if(k=="csjj_avg_price"){
|
||||
web = "城市居家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -528,6 +530,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
@@ -592,6 +595,14 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
|
||||
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
@@ -599,8 +610,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def resultMap = [:]
|
||||
map.each {key, value->
|
||||
if(value && value.size()>1){
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
|
||||
if(value && value.size()>1){ //csjj_esf_price 国信达成交
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price && !value.csjj_esf_price){
|
||||
def price = statMap.get(key)
|
||||
if(price){
|
||||
def priceMap = [:]
|
||||
|
||||
+69
-5
@@ -32,6 +32,10 @@ class TradeStatLog {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
@@ -40,8 +44,10 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
|
||||
task.exportTrade2()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -499,8 +505,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2023-08-01"
|
||||
def endDate = "2023-10-01"
|
||||
def startDate = "2024-04-01"
|
||||
def endDate = "2024-06-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
@@ -584,6 +590,36 @@ class TradeStatLog {
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
|
||||
def exportTrade2(){
|
||||
|
||||
// def startDate = "2024-06-01"
|
||||
// def endDate = "2024-07-01"
|
||||
def ids = findAllDealIds();
|
||||
println ids.size()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
|
||||
if(ids.contains(d['A'])){
|
||||
d['V'] = "供"
|
||||
}
|
||||
if(d['S'] && !d['W']){
|
||||
def dist = CommonDatautils.findDistrictById(d['S'])
|
||||
if(!dist.name_cat && dist.name_cat!=0){
|
||||
d['W'] = "name_cat过滤"
|
||||
}
|
||||
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
|
||||
d['W'] = "小区过滤"
|
||||
}
|
||||
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
|
||||
def map = findDealByCommId(d['B'], d['S'])
|
||||
def count = map.get(key)
|
||||
if(count>=2){
|
||||
d['W'] = "重复"
|
||||
}
|
||||
}
|
||||
println d['A']
|
||||
})
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
@@ -637,11 +673,39 @@ class TradeStatLog {
|
||||
return trades
|
||||
}
|
||||
|
||||
def findAllDealIds(){
|
||||
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
|
||||
def ids = []
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.gujia_bds_mdl_id)){
|
||||
ids << r.gujia_bds_mdl_id
|
||||
}
|
||||
})
|
||||
return ids
|
||||
}
|
||||
|
||||
def findDealByCommId(city, commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_id", commid)
|
||||
q.put("city", city)
|
||||
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def count = map.get(key)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(key, count+1)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findDealIds(city, startDate, endDate){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def ids = []
|
||||
def detailInfos = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.dealid)){
|
||||
ids << r.dealid
|
||||
|
||||
|
||||
+111
-40
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def size = regionDatas.size()
|
||||
final CountDownLatch latch = new CountDownLatch(size)
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
for(def i = 0;i<regionDatas.size();i++){
|
||||
final def arr = regionDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
totalCount.addAndGet(1)
|
||||
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
|
||||
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
|
||||
if(row){
|
||||
rets << row
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// };
|
||||
// })
|
||||
}
|
||||
latch.await()
|
||||
// latch.await()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
|
||||
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def allDistRegionPreMap = [:]
|
||||
def allLastMonthMap = [:]
|
||||
def allLastYearMap = [:]
|
||||
def allCcbPriceMap = [:]
|
||||
def allCcbPriceCaseCountMap = [:]
|
||||
for(def city : citys) {
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
|
||||
allDistricts.addAll(districts)
|
||||
|
||||
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
allPriceMap = allPriceMap + priceMap
|
||||
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
// allPriceMap = allPriceMap + priceMap
|
||||
|
||||
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
|
||||
allRentMap = allRentMap + rentMap
|
||||
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
|
||||
allDealCountMap = allDealCountMap + dealCountMap
|
||||
|
||||
|
||||
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
|
||||
|
||||
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceMap = allCcbPriceMap + datasMap
|
||||
|
||||
|
||||
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
|
||||
|
||||
def cityRankMap = findCommunityCityRank(datas)
|
||||
allCityRankMap = allCityRankMap + cityRankMap
|
||||
|
||||
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
allLastYearMap = allLastYearMap + lastYearMonthMap
|
||||
}
|
||||
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(cityLists){
|
||||
rets.addAll(cityLists)
|
||||
}
|
||||
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def totalCount = new AtomicLong()
|
||||
def distCount = districts.size()
|
||||
final CountDownLatch latch = new CountDownLatch(distCount)
|
||||
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(row){
|
||||
rets << row
|
||||
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
|
||||
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def sellDataType = 1
|
||||
def rentDataType = 1
|
||||
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
//房源查询是否按建行均价筛选房源
|
||||
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
def sell_avg_price = null
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
def sell_count = 0
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
|
||||
|
||||
if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
|
||||
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
}
|
||||
}
|
||||
if(sell_count){
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(sellMap){
|
||||
sell_count = sellMap.avg_price.size()
|
||||
if(sell_count>3){
|
||||
sell_avg_price = avgValue(sellMap.avg_price)
|
||||
if(sellMap){
|
||||
sell_avg_area = avgValue(sellMap.area)
|
||||
sell_avg_total = avgValue(sellMap.total_price)
|
||||
}else{
|
||||
sell_count = 0
|
||||
sellDataType = null
|
||||
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// if(sell_count>3){
|
||||
// if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def rent_avg_price = null
|
||||
def rent_avg_area = null
|
||||
def rent_avg_total = null
|
||||
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
rentDataType = null
|
||||
}
|
||||
}
|
||||
if(!sell_avg_price){
|
||||
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
|
||||
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
|
||||
// if(!sell_avg_price){
|
||||
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
// }
|
||||
if(!rent_avg_price){
|
||||
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
|
||||
def avgPrice = null
|
||||
if(region){
|
||||
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
|
||||
}else{
|
||||
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
|
||||
}
|
||||
|
||||
|
||||
if (sell_count < 10) {
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
sellStartDate = startDate
|
||||
}
|
||||
if (sell_count) {
|
||||
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
|
||||
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
|
||||
}
|
||||
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return datas;
|
||||
}
|
||||
|
||||
def findCommunityPriceByCityMap(cityCode, month){
|
||||
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCommunityPriceCountByCityMap(cityCode, month){
|
||||
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
if(r.gujiabdsrt3mlgcasenum){
|
||||
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
|
||||
}
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCcbCityPrice(cityCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def avgPrice = null
|
||||
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return regionPriceMap;
|
||||
}
|
||||
|
||||
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
|
||||
def price = null
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
price = r.gujia_bds_clc_avprc
|
||||
})
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
def districtCityPrictRate(cityPrice, datas){
|
||||
def distCityRateMap = [:]
|
||||
|
||||
+7
-7
@@ -159,7 +159,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
|
||||
}
|
||||
@@ -246,7 +246,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
|
||||
}
|
||||
@@ -304,7 +304,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
|
||||
}
|
||||
@@ -362,7 +362,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
|
||||
}
|
||||
@@ -425,7 +425,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
|
||||
}
|
||||
@@ -504,7 +504,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
|
||||
}
|
||||
@@ -590,7 +590,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
rets << ret
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
|
||||
}
|
||||
|
||||
+119
-41
@@ -4,17 +4,15 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.sql.ResultSetMetaData
|
||||
|
||||
/**
|
||||
* 建行融合数据导出
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
@@ -69,6 +67,12 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
exportBigdataCityPrice(queryDate)
|
||||
exportBigdataDistPrice(queryDate)
|
||||
exportBigdataCommPrice(queryDate)
|
||||
// 新增两张数据表
|
||||
// avm_evaluate_rate_city_gj_h
|
||||
// avm_evaluate_rate_comm_gj_h
|
||||
exportEvaluateRateCity(queryDate)
|
||||
exportEvaluateRateComm(queryDate)
|
||||
|
||||
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
|
||||
// exportCommunityBasePrice(lastMonth)
|
||||
// exportCommunityPdescribeGj(lastMonth)
|
||||
@@ -142,7 +146,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataCommunityGJRH(){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -217,7 +221,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -228,6 +232,80 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
// avm_evaluate_rate_city_gj_h gujia_clc_yrmo_val =202402 and gujia_clc_yrmo_val<=202403 gujia_clc_yrmo_val>=to_date('${startDateStr}','yyyy-MM-dd') and gujia_clc_yrmo_val<=to_date('${endDateStr}','yyyy-MM-dd')
|
||||
def exportEvaluateRateCity(queryDate){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_bld_tp_val
|
||||
data << r.gujia_bld_tp_val_coef
|
||||
data << r.gujia_clc_yrmo_val
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
|
||||
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportEvaluateRateComm(queryDate){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' ";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_bds_id
|
||||
data << r.comm_nm
|
||||
data << r.gujia_clc_yrmo_val
|
||||
data << r.basic_price
|
||||
data << r.area_avg
|
||||
data << r.unitprc_avg
|
||||
data << r.comm_type
|
||||
data << r.area_avg_coef
|
||||
data << r.unitprc_avg_coef
|
||||
data << r.basic_price_coef
|
||||
data << r.buildarea_coef
|
||||
data << r.type_coef
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
|
||||
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
@@ -249,7 +327,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -278,7 +356,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -306,7 +384,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -344,7 +422,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
});
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -374,7 +452,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -407,7 +485,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -439,7 +517,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -474,7 +552,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -498,7 +576,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
@@ -598,7 +676,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -638,7 +716,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -909,7 +987,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -920,7 +998,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildingFJW(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -994,7 +1072,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1006,7 +1084,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportThirdPartyPrice2(price_date, new_dists) {
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1054,7 +1132,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgThirdPartyPrice2(price_date, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1137,7 +1215,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1183,7 +1261,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1226,7 +1304,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1241,7 +1319,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1315,7 +1393,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1382,7 +1460,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1423,7 +1501,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1459,7 +1537,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgBigdataSupplierStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1492,7 +1570,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataSupplierStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1531,7 +1609,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataModelingBase(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1574,7 +1652,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgBigdataModelingBase(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1611,7 +1689,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCityPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1659,7 +1737,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataDistPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1706,7 +1784,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCommPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1767,7 +1845,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1778,7 +1856,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataForsale(publishDate, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1835,7 +1913,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgBigdataForsale(publishDate,endDate, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -2020,7 +2098,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
def dataCodeFileName = "/home/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
|
||||
+651
-36
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
@@ -8,11 +9,18 @@ import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.client.MongoCursor
|
||||
import com.mongodb.client.model.Accumulators
|
||||
import com.mongodb.client.model.Aggregates
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
@@ -40,6 +48,8 @@ import java.util.zip.GZIPInputStream
|
||||
|
||||
class CommonDatautils {
|
||||
|
||||
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
@@ -108,6 +118,9 @@ class CommonDatautils {
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -265,7 +278,7 @@ class CommonDatautils {
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","泰瑞家园")
|
||||
// q.put("name","托斯卡纳庄园")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -274,7 +287,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -1175,6 +1188,35 @@ class CommonDatautils {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findAllStatusDistrict(String city,String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("region", region);
|
||||
q.put("name", name);
|
||||
q.put("cat", cat);
|
||||
|
||||
def rets = []
|
||||
def cursor = SEAWEED_MONGO_COLL.find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
rets << obj;
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def findAllCount(String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", cat);
|
||||
q.put("status", 0)
|
||||
return SEAWEED_MONGO_COLL.count(q)
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrictsByName(String city, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -1435,6 +1477,28 @@ class CommonDatautils {
|
||||
return obj;
|
||||
}
|
||||
|
||||
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
|
||||
BasicDBObject query = new BasicDBObject();
|
||||
query.putAt(QueryOperators.IN, [name]);
|
||||
q.put("mapping_alias", query);
|
||||
|
||||
def dbcursor = SEAWEED_COLL.find(q);
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findSeaweeds(city, cat, name_cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put('city',city)
|
||||
@@ -1537,6 +1601,32 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def findAllStatusDistrictByAlias(def city, def region, def name, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
|
||||
def names = [];
|
||||
names << name;
|
||||
BasicDBObject query = new BasicDBObject();
|
||||
query.putAt(QueryOperators.IN, names);
|
||||
q.put("alias", query);
|
||||
|
||||
|
||||
def dsCursor = getDataStoreConnection().find(q);
|
||||
def ret = [];
|
||||
for(def obj : dsCursor){
|
||||
ret << obj;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static def findDistrictByAddress(def city, def region, def address, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
@@ -1641,7 +1731,34 @@ class CommonDatautils {
|
||||
}
|
||||
return regionMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def getAllCityFullRegions(){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("category","region");
|
||||
//queryCondition.put("show",1);
|
||||
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
|
||||
def regionMap = [:];
|
||||
for(DBObject obj : dbCursor){
|
||||
def city = obj.get("city");
|
||||
def region = obj.get("name");
|
||||
def pattern = obj.get("pattern");
|
||||
if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) {
|
||||
def r_map = regionMap.get(city);
|
||||
if(!r_map){
|
||||
r_map = [:]
|
||||
regionMap.put(city, r_map);
|
||||
}
|
||||
|
||||
if(!pattern){
|
||||
pattern = [];
|
||||
pattern << region;
|
||||
}
|
||||
r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]);
|
||||
}
|
||||
}
|
||||
return regionMap;
|
||||
}
|
||||
|
||||
|
||||
public static def getRegionsByCity(def city){
|
||||
@@ -1737,39 +1854,6 @@ class CommonDatautils {
|
||||
}
|
||||
return priceMap;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend, cat){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
|
||||
if(cat == "district"){
|
||||
BasicDBObject queryObj = new BasicDBObject();
|
||||
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
|
||||
queryCondition.put("estate_type", queryObj);
|
||||
}else if(cat == "office"){
|
||||
queryCondition.put("estate_type", "写字楼");
|
||||
}else if(cat == "shop"){
|
||||
queryCondition.put("estate_type", "商铺");
|
||||
}
|
||||
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
public static def queryFirstDistrictPrice(city, region, name){
|
||||
@@ -1784,11 +1868,63 @@ class CommonDatautils {
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
dbo = cur.next()
|
||||
if(dbo.value_type == 1){
|
||||
dbo.cal_type = "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
dbo.cal_type = "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
dbo.cal_type = "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
dbo.cal_type = "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
dbo.cal_type = "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
dbo.cal_type = "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
dbo.cal_type = "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
dbo.cal_type = "无方式"
|
||||
}else{
|
||||
dbo.cal_type = ""
|
||||
}
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend, cat){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
|
||||
if(cat == "district"){
|
||||
BasicDBObject queryObj = new BasicDBObject();
|
||||
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
|
||||
queryCondition.put("estate_type", queryObj);
|
||||
}else if(cat == "office"){
|
||||
queryCondition.put("estate_type", "写字楼");
|
||||
}else if(cat == "shop"){
|
||||
queryCondition.put("estate_type", "商铺");
|
||||
}
|
||||
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
@@ -3764,6 +3900,135 @@ class CommonDatautils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def findHouses2(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(true);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("cat", cat)).must(date_range).must(area_range)
|
||||
|
||||
if(region){
|
||||
queryBuilder.must(QueryBuilders.termQuery("region", region))
|
||||
}
|
||||
|
||||
if(name || alias){
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
def avgPrices = []
|
||||
def areas = []
|
||||
def totalPrices = []
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(cat == "rent"){
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
}
|
||||
avgPrices << seawater.avg_price
|
||||
areas << seawater.area
|
||||
totalPrices << seawater.total_price
|
||||
}
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
if(areas){
|
||||
retMap.avg_price = avgPrices
|
||||
retMap.area = areas
|
||||
retMap.total_price = totalPrices
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public static boolean isVilla(Double area, String name, Integer height, Integer room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
}else if(height != null){
|
||||
if (height <= 3) {
|
||||
if (area > 200){
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
@@ -3919,6 +4184,29 @@ class CommonDatautils {
|
||||
return avg_price
|
||||
}
|
||||
|
||||
static def findMatchTrades(city, region, name, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
@@ -3993,6 +4281,32 @@ class CommonDatautils {
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
static def findLogs(user_id, requestUri, startDate, endDate){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", user_id)).must(QueryBuilders.termQuery("request_uri", requestUri))
|
||||
if(startDate){
|
||||
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
|
||||
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
|
||||
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
|
||||
queryBuilder.must(date_range_cdate);
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
|
||||
def map = houses[k];
|
||||
if(map.parameter_map){
|
||||
def pMap = map.parameter_map;
|
||||
if((pMap.keyword && pMap.keyword.contains("紫金山蘭园")) || (pMap.name && pMap.name.contains("紫金山蘭园"))){
|
||||
println map
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def findNewAmapDetailBySql(city){
|
||||
String sql = "select * from amap_poi where cityname='${city}' and status=0"
|
||||
def rets = []
|
||||
@@ -4219,6 +4533,23 @@ class CommonDatautils {
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("xingfuli_esf_price") && db.xingfuli_esf_price && db.xingfuli_esf_price>3000){
|
||||
dataMap.put("xingfuli_esf_price", db.xingfuli_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
@@ -4251,6 +4582,28 @@ class CommonDatautils {
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def findAllTownRanges(){
|
||||
def q = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("town", 1)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def lists = townMap.get(d.city)
|
||||
if(!lists){
|
||||
lists = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.region
|
||||
t.town = d.town
|
||||
lists << t
|
||||
townMap.put(d.city, lists)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
@@ -4285,6 +4638,148 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryFullNameByCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","城市")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def city = RegionData.formatCity(d.city)
|
||||
if(city){
|
||||
cityMap.put(city, d)
|
||||
}else{
|
||||
//println d.city +" 格式化不出"
|
||||
}
|
||||
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
static def queryDistNameByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
keys.put("district_short",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.mzb_district_name){
|
||||
def regions = regionMap.get(d.city)
|
||||
if(!regions){
|
||||
regions = []
|
||||
}
|
||||
regions << d
|
||||
regionMap.put(d.city, regions)
|
||||
}else{
|
||||
println d.province+" "+d.district +" 格式化不出民政部区域"
|
||||
}
|
||||
}
|
||||
|
||||
println "区域加载完成"
|
||||
return regionMap
|
||||
}
|
||||
|
||||
static def queryCommunityByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
||||
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "乡镇街道 加载完成"
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryTownByCity(city){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
static def queryNewTownByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.district
|
||||
t.town = d.street
|
||||
list << t
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -4312,6 +4807,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryDistNameCityByMapping(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.district_short && d.city_short){
|
||||
retMap.put(d.city_short+"#"+d.district_short, d)
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -4337,6 +4847,111 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def findHouseAvgPrice(city, region , start, end){
|
||||
def prices = []
|
||||
def sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date > ${start} and p_date<${end} and (root='lianjia.com' or root='ke.com')"
|
||||
if(region){
|
||||
sql = sql + " and region='${region}'"
|
||||
}
|
||||
ElasticUtils.executeSqlQuery(sql, {
|
||||
prices << it.avg_price
|
||||
})
|
||||
if(prices && prices.size()>3){
|
||||
def avg_price = (int)(prices.sum()/prices.size())
|
||||
println region + " "+ prices.size() +" "+avg_price +" "+start+" "+end
|
||||
return avg_price
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
def road = ""
|
||||
def name = ""
|
||||
def type = ""
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if(addressComponent.streetNumber){
|
||||
if(addressComponent.streetNumber.street){
|
||||
road = addressComponent.streetNumber.street
|
||||
}
|
||||
if(addressComponent.streetNumber.number){
|
||||
road = road + addressComponent.streetNumber.number
|
||||
}
|
||||
}
|
||||
|
||||
if(addressComponent.building){
|
||||
if(addressComponent.building.name){
|
||||
name = addressComponent.building.name
|
||||
}
|
||||
if(addressComponent.building.type){
|
||||
type = addressComponent.building.type
|
||||
}
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
map.road = road
|
||||
map.type = type
|
||||
map.name = name
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
static def saveOptLog(city, region, name, cat, opt, user_name){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category",cat);
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", opt)
|
||||
obj.put("user_name", user_name)
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+15
-9
@@ -44,12 +44,17 @@ class DistrictHouseBatch extends BaseBatch {
|
||||
public void process(Object args) {
|
||||
|
||||
// DBObject q = new BasicDBObject();
|
||||
// q.put("_id", new ObjectId("6501174f6f5638447b264189"));
|
||||
// def obj = CommonDatautils.getHouseDataStoreConnection().findOne(q)
|
||||
// obj.put("c_date", new Date())
|
||||
// CommonDatautils.getHouseDataStoreConnection().save(obj)
|
||||
// q.put("cat", "sell");
|
||||
// q.put("city","常州")
|
||||
// q.put("district_name","清河苑一区")
|
||||
// def dbcursor = CommonDatautils.getHouseDataStoreConnection().find(q)
|
||||
// for(def obj : dbcursor){
|
||||
// obj.put("c_date", new Date())
|
||||
// CommonDatautils.getHouseDataStoreConnection().save(obj)
|
||||
// }
|
||||
|
||||
// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","新城帝景南区_武进_常州")).each{
|
||||
|
||||
// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","清河苑一区_溧阳_常州")).each{
|
||||
// println(it)
|
||||
// }
|
||||
|
||||
@@ -62,14 +67,15 @@ class DistrictHouseBatch extends BaseBatch {
|
||||
c++
|
||||
try {
|
||||
long sn = Long.parseLong("${new Date().format('yyyyMMddHHmmssSSS')}")
|
||||
List<DBObject> districts = findDistrictByCity(city)
|
||||
// List<DBObject> districts = findDistrictByCity(city)
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"])
|
||||
Map<String, Long> sellCountMap = facetDistrictInHouse("sell", city);
|
||||
Map<String, Long> rentCountMap = facetDistrictInHouse("rent", city);
|
||||
println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
|
||||
for(DBObject district : districts) {
|
||||
def names = []
|
||||
names << district.name
|
||||
if(district.alias && districts.size()>0){
|
||||
if(district.alias && districts.alias.size()>0){
|
||||
for(def name : district.alias){
|
||||
if(!names.contains(name)){
|
||||
names << name
|
||||
@@ -88,11 +94,11 @@ class DistrictHouseBatch extends BaseBatch {
|
||||
rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0
|
||||
}
|
||||
}
|
||||
|
||||
saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn)
|
||||
}
|
||||
|
||||
HOUSE_COLL.remove(new BasicDBObject("city", city).append('sn', new BasicDBObject('$ne', sn)))
|
||||
println c + " " + city
|
||||
}catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
@@ -151,7 +157,7 @@ class DistrictHouseBatch extends BaseBatch {
|
||||
q.put('city', city)
|
||||
q.put('cat', 'district')
|
||||
q.put('visible', 0)
|
||||
// q.put("name","新城帝景")
|
||||
q.put("name","清河苑")
|
||||
DSCursor cur = SEAWEED.find(q, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
|
||||
+25
-12
@@ -7,6 +7,7 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -74,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
|
||||
def day = new Date().getDate()
|
||||
def startMDate = DateUtils.truncateDay(new Date()-1)
|
||||
def startMDate = DateUtils.truncateDay(new Date()-7)
|
||||
def endMDate = DateUtils.truncateDay(new Date())
|
||||
|
||||
def index = 0
|
||||
@@ -144,7 +145,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def key = dist.region +"##" + dist.name
|
||||
def type = dist.estate_type2=="别墅"?"别墅":"小区"
|
||||
def key = dist.region +"##" + dist.name+"##"+ type
|
||||
|
||||
def weekendPriceObj = weekendPriceMap.get(key)
|
||||
if(weekendPriceObj){
|
||||
@@ -153,8 +155,13 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def lastYearPriceObj = lastYearPriceMap.get(key)
|
||||
double mom = 0d
|
||||
|
||||
if(lastMonthPriceObj) {
|
||||
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
|
||||
if(lastMonthPriceObj && lastMonthPriceObj.avg_price>0) {
|
||||
try{
|
||||
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
|
||||
}catch(Exception e){
|
||||
println key+" 出错................."
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def curPrice = weekendPriceObj.avg_price
|
||||
@@ -192,7 +199,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
curPrice = 0
|
||||
}
|
||||
if(curPriceObj){
|
||||
if(curPrice!=weekendPriceObj.avg_price){
|
||||
if(curPrice!=curPriceObj.avg_price){
|
||||
def data = new BasicDBObject()
|
||||
data.put("m_date", new Date())
|
||||
data.put("commid", dist._id.toString())
|
||||
@@ -257,13 +264,13 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(city == "¸£ÖÝ"){
|
||||
if(d.avg_price_2b && d.value_type!=1){
|
||||
d.avg_price = d.avg_price_2b
|
||||
if(!priceMap.containsKey(d.region +"##"+d.district_name+"##"+d.estate_type)){
|
||||
if(city == "福州"){
|
||||
if(d.avg_price_2b && d.value_type!=1){
|
||||
d.avg_price = d.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d)
|
||||
}
|
||||
}
|
||||
return priceMap
|
||||
@@ -276,7 +283,12 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def dbcursor = DISTRICT_SELL_STAT.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
if(d.estate_type=="别墅"){
|
||||
priceMap.put(d.region +"##"+d.district_name+"##别墅", d)
|
||||
}else{
|
||||
priceMap.put(d.region +"##"+d.district_name+"##小区", d)
|
||||
}
|
||||
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
@@ -355,4 +367,5 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
// }
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+28
-6
@@ -9,6 +9,7 @@ import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.AMQP
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
@@ -29,9 +30,10 @@ class FzDataResultStat {
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
task.regionPriceTrendRun()
|
||||
// task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
// task.testNewPrice()
|
||||
}
|
||||
|
||||
def margeExcel(){
|
||||
@@ -56,7 +58,7 @@ class FzDataResultStat {
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||
@@ -80,7 +82,7 @@ class FzDataResultStat {
|
||||
}
|
||||
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def list2 = findFzDistrictStat(lastMonth)
|
||||
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
|
||||
def map2 = [:]
|
||||
for(def obj : list2) {
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
@@ -163,7 +165,7 @@ class FzDataResultStat {
|
||||
|
||||
|
||||
def exportRegion(city, months){
|
||||
def regions = ["뮬짜","꾑�","憩쉭","쐈갛","쯩棺","낀있"];
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"];
|
||||
def datas =[]
|
||||
def header = ["城市", "区域"]
|
||||
for(def m : months){
|
||||
@@ -206,8 +208,12 @@ class FzDataResultStat {
|
||||
DBObject dbo = cur.next()
|
||||
prices << dbo.get("avg_price");
|
||||
}
|
||||
def avg_price = (int)(prices.sum()/prices.size())
|
||||
return avg_price;
|
||||
if(prices){
|
||||
def avg_price = (int)(prices.sum()/prices.size())
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -293,4 +299,20 @@ class FzDataResultStat {
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def testNewPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d->
|
||||
if(d['A']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region",d['B'])
|
||||
q.put("name",d['C'])
|
||||
q.put("month", 202402)
|
||||
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
||||
if(obj){
|
||||
d['S'] = obj.avg_price
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+120
-1
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -19,7 +20,72 @@ class FzDistrictMatch {
|
||||
def task = new FzDistrictMatch()
|
||||
// task.run();
|
||||
// task.addPrice()
|
||||
task.addThirdPrice()
|
||||
// task.addThirdPrice()
|
||||
|
||||
task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
|
||||
// task.longyanPrice()
|
||||
}
|
||||
|
||||
def longyanPrice(){
|
||||
// def dists = CommonDatautils.findAllDistricts("龙岩","district", ["city","region","name","estate_type2"])
|
||||
// def rets = []
|
||||
// rets << ["Id", "城市","区域", "名称", "物业类型", "上个月价格toB","上个月价格","最新价格toB","最新价格"]
|
||||
// for(def d : dists){
|
||||
// def ret = []
|
||||
// ret << d._id.toString()
|
||||
// ret << d.city
|
||||
// ret << d.region
|
||||
// ret << d.name
|
||||
// ret << d.estate_type2
|
||||
// def obj2 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-03-10"), 1)
|
||||
// if(obj2){
|
||||
// ret << obj2.avg_price_2b
|
||||
// ret << obj2.avg_price
|
||||
// }
|
||||
//
|
||||
// def obj1 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-04-07"), 1)
|
||||
// if(obj1){
|
||||
// ret << obj1.avg_price_2b
|
||||
// ret << obj1.avg_price
|
||||
// }
|
||||
// rets << ret
|
||||
// }
|
||||
// ExcelUtils.write("D:\\数据\\福州\\龙岩价格对比.xlsx", rets, "龙岩")
|
||||
|
||||
|
||||
addThirdPrice("D:\\数据\\福州\\龙岩价格对比.xlsx")
|
||||
}
|
||||
|
||||
def addThirdPrice2(path){
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(row['A'], row['B'], row['C'], 202312, 202402)
|
||||
def priceMap = cityMap.get(row['A']+"##"+row['B']+"##"+row['C'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['J'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(!row['J'] && priceMap.containsKey("beike_esf_price")){
|
||||
row['J'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("maitian_esf_price")){
|
||||
row['K'] = priceMap.get("maitian_esf_price")
|
||||
}
|
||||
}
|
||||
|
||||
def tradeAvgPrice = CommonDatautils.findMatchTrades(row['A'], row['B'], row['C'], "2023-08-01", "2024-01-31");
|
||||
if(tradeAvgPrice){
|
||||
row['L'] = (int)tradeAvgPrice
|
||||
}
|
||||
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['A'], row['B'], row['C'])
|
||||
if(priceObj){
|
||||
row['M'] = priceObj.avg_price_2b
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addThirdPrice(){
|
||||
@@ -173,4 +239,57 @@ class FzDistrictMatch {
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(d['C'] && d['C']=="新罗"){
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(d['B'], d['C'], d['D'], 202312, 202403)
|
||||
def priceMap = cityMap.get(d['B']+"##"+d['C']+"##"+d['D'])
|
||||
|
||||
if(priceMap && priceMap.zhugezhaofang_esf_price){
|
||||
d['J'] = priceMap.zhugezhaofang_esf_price
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.anjuke_esf_price){
|
||||
d['K'] = priceMap.anjuke_esf_price
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.soufang_esf_price) {
|
||||
d['L'] = priceMap.soufang_esf_price
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.lianjia_esf_price){
|
||||
d['M'] = priceMap.lianjia_esf_price
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.cityhouse_esf_price){
|
||||
d['N'] = priceMap.cityhouse_esf_price
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.zhongyuan_esf_price){
|
||||
d['O'] = priceMap.zhongyuan_esf_price
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.xingfuli_esf_price){
|
||||
d['P'] = priceMap.xingfuli_esf_price
|
||||
}
|
||||
if(priceMap && priceMap.tuboshi_esf_price){
|
||||
d['Q'] = priceMap.tuboshi_esf_price
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.get("5i5j_esf_price")){
|
||||
d['R'] = priceMap.get("5i5j_esf_price")
|
||||
}
|
||||
|
||||
def sellMap = CommonDatautils.findHouses2(d['B'], d['C'], "sell", 20231001, 20240407, d['D'], null, d['E'])
|
||||
if(sellMap && sellMap.avg_price){
|
||||
if(sellMap.avg_price.size()>=3){
|
||||
d['S'] = Math.round(sellMap.avg_price.sum() / sellMap.avg_price.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+5
-5
@@ -20,16 +20,16 @@ class AddAvgpriceBatch {
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static {
|
||||
citys = ["ÐìÖÝ"]//CommonDatautils.findAllCity()
|
||||
citys = ["Ò˲ý"]//CommonDatautils.findAllCity()
|
||||
println "加载城市完成:"+citys.size()
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
def task = new AddAvgpriceBatch();
|
||||
// task.run1();
|
||||
task.run1();
|
||||
// task.test()
|
||||
// task.sendPriceCommand()
|
||||
task.sendNewDistPrice()
|
||||
// task.sendNewDistPrice()
|
||||
}
|
||||
|
||||
def sendNewDistPrice(){
|
||||
@@ -107,7 +107,7 @@ class AddAvgpriceBatch {
|
||||
println "------------------------------------------------------------"
|
||||
println "${city}小区量:"+districtKeysMap.size()
|
||||
if(districtKeysMap){
|
||||
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26"))
|
||||
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2024-04-07"))
|
||||
println "【2023-03-26】共计均价小区:"+lastWeekendStats.size()
|
||||
//库中小区没有计算过均价的小区
|
||||
def districtKeys = districtKeysMap.keySet();
|
||||
@@ -118,7 +118,7 @@ class AddAvgpriceBatch {
|
||||
def array = disKey.split("##");
|
||||
def estateType2 = districtKeysMap.get(disKey)
|
||||
//发重算均价命令
|
||||
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd'))
|
||||
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2024-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2024-04-07','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+44
-6
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.Community
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -12,7 +14,41 @@ class CitySellStatTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new CitySellStatTest();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.cityPriceRun()
|
||||
}
|
||||
|
||||
def cityPriceRun(){
|
||||
def cityObjs = CommonDatautils.findCitys();
|
||||
def rets = []
|
||||
rets << ["序号","城市","城市级别","城市均价起始月","起始均价","城市均价最新月","最新均价"]
|
||||
def index = 1
|
||||
for(def o : cityObjs){
|
||||
def ret = []
|
||||
ret << index
|
||||
ret << o.display
|
||||
ret << o.new_city_grade
|
||||
def row1 = findCityMonthPrice(o.display, 1)
|
||||
if(row1){
|
||||
ret << row1.date
|
||||
ret << row1.avg_price
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
def row2 = findCityMonthPrice(o.display, -1)
|
||||
if(row2){
|
||||
ret << row2.date
|
||||
ret << row2.avg_price
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
rets << ret
|
||||
index++
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\其它\\全国城市均价.xlsx", rets, "城市均价")
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -25,18 +61,20 @@ class CitySellStatTest {
|
||||
},0)
|
||||
}
|
||||
|
||||
def findFirstMonth(def city){
|
||||
def findCityMonthPrice(def city, def asc){
|
||||
def q = new BasicDBObject();
|
||||
q.put("city", city)
|
||||
def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("_id",1)).limit(1)
|
||||
def month;
|
||||
def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("date", asc)).limit(1)
|
||||
def obj = null
|
||||
for(def o : dbcursor){
|
||||
month = o.date
|
||||
obj = o
|
||||
}
|
||||
return month
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findTotalMonth(def city, def month){
|
||||
def q = new BasicDBObject();
|
||||
q.put("city", city)
|
||||
|
||||
+15
-11
@@ -23,18 +23,22 @@ class EsSearchUtils {
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
def m = [:]
|
||||
m.city = data.cityname
|
||||
m.region = data.adname
|
||||
m.name = data.name
|
||||
m.address = data.address
|
||||
m.type = data.type
|
||||
m.code = data.typecode
|
||||
def mapName = data.name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
def newName = name.replaceAll('[·\\(\\)()]|·', '')
|
||||
if(newName == mapName){
|
||||
def m = [:]
|
||||
m.city = data.cityname
|
||||
m.region = data.adname
|
||||
m.name = data.name
|
||||
m.address = data.address
|
||||
m.type = data.type
|
||||
m.code = data.typecode
|
||||
|
||||
def location = data.location
|
||||
m.lng = location.lon
|
||||
m.lat = location.lat
|
||||
rets << m
|
||||
def location = data.location
|
||||
m.lng = location.lon
|
||||
m.lat = location.lat
|
||||
rets << m
|
||||
}
|
||||
})
|
||||
|
||||
return rets;
|
||||
|
||||
@@ -238,6 +238,7 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
static def getValue(def val){
|
||||
val = val.replace(",","")
|
||||
val = getNumber(val);
|
||||
if(val){
|
||||
if(val.indexOf(".")>-1 && !val.toString().endsWith(".0")){
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class GreenBuild {
|
||||
|
||||
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new GreenBuild()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAll()
|
||||
def rets = match(datas)
|
||||
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
|
||||
}
|
||||
|
||||
def match(datas){
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
|
||||
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
def resultMap = jsonObject.get("result")
|
||||
data.match_id = resultMap.get("id").toString()
|
||||
data.match_city = resultMap.get("city")
|
||||
data.match_district = resultMap.get("district")
|
||||
data.match_name = resultMap.get("name")
|
||||
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
|
||||
}else{
|
||||
rets << [data._id.toString(), data.city, data.district, data.name]
|
||||
}
|
||||
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await();
|
||||
return rets
|
||||
}
|
||||
|
||||
def findAll(){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -13,12 +13,18 @@ import net.sf.json.groovy.JsonSlurper
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class OpenApiUtils {
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
|
||||
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
}
|
||||
if(suggestName){
|
||||
url = url +"&suggestName="+suggestName
|
||||
}
|
||||
if(suggestDistrict){
|
||||
url = url +"&suggestDistrict="+suggestDistrict
|
||||
}
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
|
||||
@@ -12,8 +12,8 @@ class OpenUserBatch {
|
||||
def task = new OpenUserBatch();
|
||||
// task.run();
|
||||
// task.test();
|
||||
// task.addShenZhenFjtUser();
|
||||
task.addFujianFjtUser()
|
||||
task.addShenZhenFjtUser();
|
||||
// task.addFujianFjtUser()
|
||||
}
|
||||
|
||||
def test(){
|
||||
@@ -46,20 +46,20 @@ class OpenUserBatch {
|
||||
}
|
||||
|
||||
def addShenZhenFjtUser(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
|
||||
def i = 0;
|
||||
for(def data : datas){
|
||||
i++;
|
||||
def id = 252608 + i;
|
||||
def id = 252893 + i;
|
||||
BigDecimal bd = new BigDecimal(data[3].toString());
|
||||
String mobile = bd.toPlainString();
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
|
||||
onLineSqlInstance.execute(sql);
|
||||
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
|
||||
onLineSqlInstance.execute(sql2)
|
||||
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
|
||||
onLineSqlInstance.execute(sql3)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class YouDaDataParser {
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
|
||||
task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+28
-1
@@ -31,7 +31,34 @@ class CityStatExport {
|
||||
// task.regionStat()
|
||||
// task.stat()
|
||||
// task.houseStat()
|
||||
task.addFieldStat()
|
||||
// task.addFieldStat()
|
||||
|
||||
task.exportRegionAvgPrice()
|
||||
}
|
||||
|
||||
def exportRegionAvgPrice(){
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"];
|
||||
def add = -9;
|
||||
def months = []
|
||||
while(add<=0){
|
||||
months << DateUtils.addMonth(new Date(), add)
|
||||
add++
|
||||
}
|
||||
def rets = []
|
||||
rets << months
|
||||
for(def region : regions){
|
||||
def row = []
|
||||
row << region
|
||||
for(def i=0; i<months.size()-1;i++){
|
||||
def start = months[i].replace("-","").toInteger()
|
||||
def end = months[i+1].replace("-","").toInteger()
|
||||
def avgprice = CommonDatautils.findHouseAvgPrice("福州", region, start, end)
|
||||
row << avgprice
|
||||
}
|
||||
rets.add(row)
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州区域及城市走势_2024-03.xlsx", rets, "房源均价")
|
||||
}
|
||||
|
||||
def addFieldStat(){
|
||||
|
||||
+10
-1
@@ -33,7 +33,16 @@ class CityStat {
|
||||
// task.run6()
|
||||
// task.run7()
|
||||
// task.run8()
|
||||
task.dealPriceStat()
|
||||
// task.dealPriceStat()
|
||||
task.countDist()
|
||||
}
|
||||
def countDist(){
|
||||
ExcelUtils.modify("D:\\数据\\其它\\小区统计.xlsx", { d ->
|
||||
if(d['B']){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city", "region", "name"])
|
||||
d['D'] = districts.size()
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def dealPriceStat(){
|
||||
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
//郑州银行数据导出
|
||||
class ZhengzhouBlankStat {
|
||||
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def DISTRICT_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
|
||||
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
def task = new ZhengzhouBlankStat();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["郑州","洛阳","平顶山","安阳","许昌","南阳","新乡","焦作","开封","信阳","驻马店","漯河","鹤壁","周口","濮阳","三门峡","商丘","济源"]
|
||||
//statRegionDistrictCount(citys)
|
||||
// statCityProcent(citys)
|
||||
statCityHouseloudongProcent(citys)
|
||||
|
||||
}
|
||||
|
||||
def statRegionDistrictCount(citys){
|
||||
def rets = []
|
||||
rets << ["城市", "区县", "楼盘数量"]
|
||||
for(def c : citys){
|
||||
def regions = CommonDatautils.getCityRegions(c)
|
||||
|
||||
for(def r : regions){
|
||||
def count = CommonDatautils.findRegionDistrictCount("district", c, r)
|
||||
def ret = []
|
||||
ret << c
|
||||
ret << r
|
||||
ret << count
|
||||
rets << ret
|
||||
}
|
||||
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省区县小区量统计")
|
||||
}
|
||||
|
||||
def statCityProcent(citys){
|
||||
def rets = []
|
||||
rets << ["城市","数据量","地址","地址别名","别名","楼栋总数","户数","产权年限","产权性质","建筑结构","竣工时间","小区当前均价"]
|
||||
for(def c : citys){
|
||||
def dbo = CommonDatautils.findCityInfo(c)
|
||||
if(dbo){
|
||||
def detail = dbo.district_detail
|
||||
def ret = []
|
||||
ret << c
|
||||
if(detail){
|
||||
ret << dbo.existing_district_count//数据量
|
||||
ret << detail.address*100// 地址
|
||||
ret << detail.addrAlias*100// 地址别名
|
||||
ret << detail.alias*100// 别名
|
||||
ret << detail.loudongCount*100// 楼栋总数
|
||||
ret << detail.householdCount*100// 户数
|
||||
ret << detail.propertyLife*100// 产权年限
|
||||
ret << detail.houseProperty*100// 产权性质
|
||||
ret << detail.buildStructure*100// 建筑结构(按材料划分)
|
||||
ret << detail.finishBuildingDate*100// 竣工时间
|
||||
ret << dbo.price_coverage*100// 小区当前均价
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省城市字段覆盖率统计")
|
||||
}
|
||||
|
||||
def statCityHouseloudongProcent(citys){
|
||||
def rets = []
|
||||
|
||||
rets << ["城市","楼栋","单元","所在层数","房号","面积","套型","楼层"]
|
||||
for(def c : citys){
|
||||
def dbo = CommonDatautils.findCityInfo(c)
|
||||
if(dbo){
|
||||
def detail = dbo.house_loudong_detail
|
||||
def ret = []
|
||||
ret << c
|
||||
if(detail){
|
||||
ret << detail.house_num_rate*100//楼栋
|
||||
ret << detail.unit_num_rate*100// 单元
|
||||
ret << detail.room_num_rate*100// 所在层数
|
||||
ret << detail.room_num_rate*100// 房号
|
||||
ret << detail.area_rate*100// 面积
|
||||
ret << detail.room_info_rate*100// 套型
|
||||
}
|
||||
def loudong_detail = dbo.loudong_detail
|
||||
if(loudong_detail){
|
||||
ret << loudong_detail.loudong_height_rate*100// 楼层
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省房产信息字段覆盖率统计")
|
||||
}
|
||||
|
||||
}
|
||||
+247
File diff suppressed because one or more lines are too long
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.gson.JsonObject
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class BaiduBaikeTown {
|
||||
|
||||
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new BaiduBaikeTown()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
|
||||
if(!row['E'] && row['A']=="郑州经济技术开发区"){
|
||||
def q = new BasicDBObject()
|
||||
q.put("desc",new BasicDBObject('$ne', 'null'))
|
||||
q.put("name",row['A'])
|
||||
def d = dzb_COLL.findOne(q)
|
||||
def townMap = parse(d.desc, d.name)
|
||||
row['D'] = townMap.town_str1
|
||||
row['F'] = townMap.town_str2
|
||||
|
||||
// def ret = []
|
||||
// ret << ["名称", "基本信息", "目录", "全文"]
|
||||
// for(def d : dbcursor){
|
||||
// def townMap = parse(d.desc, d.name)
|
||||
// if(townMap.town){
|
||||
// println d.name+" "+townMap.town
|
||||
// }
|
||||
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
|
||||
}
|
||||
|
||||
def parse(desc, name){
|
||||
def map = [:]
|
||||
if(desc){
|
||||
def doc = Jsoup.parse(desc)
|
||||
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
|
||||
|
||||
def s = doc.toString()
|
||||
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
|
||||
def jsonObj = JSONObject.fromObject(dataStr)
|
||||
|
||||
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
|
||||
def towns = []
|
||||
if(jsonObj && jsonObj.catalog){
|
||||
def catalogs = jsonObj.catalog
|
||||
def index = null
|
||||
for(def log : catalogs){
|
||||
def curIndex = log.index
|
||||
if(log.title=="行政区划"){
|
||||
index = log.index
|
||||
}
|
||||
if(index && curIndex.toString().startsWith(index+"-")){
|
||||
towns << log.title
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
map.town_str1 = str
|
||||
if(town){
|
||||
map.town = town
|
||||
}else{
|
||||
map.town = ""
|
||||
}
|
||||
|
||||
if(towns){
|
||||
map.towns = towns.join(",")
|
||||
}else{
|
||||
map.towns = ""
|
||||
}
|
||||
|
||||
|
||||
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
|
||||
map.town_str2 = str2
|
||||
return map
|
||||
}
|
||||
map.town_str1 = ""
|
||||
map.town_str2 = ""
|
||||
map.town = ""
|
||||
map.towns = ""
|
||||
return map
|
||||
}
|
||||
}
|
||||
+1066
File diff suppressed because it is too large
Load Diff
+281
@@ -0,0 +1,281 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
class DevUpdateTestBatch {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
// task.addDevAddress(path)
|
||||
|
||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDev(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['I'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(!obj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", d['B'])
|
||||
o.put("city", d['C'])
|
||||
o.put("district", d['D'])
|
||||
o.put("name", d['A'])
|
||||
o.put("level", d['E'])
|
||||
o.put("type", d['F'])
|
||||
|
||||
if(d['G']){
|
||||
o.put("township", d['G'])
|
||||
}
|
||||
if(d['H']){
|
||||
def arr = d['H'].split(",")
|
||||
if(arr){
|
||||
o.put("alias", arr)
|
||||
}
|
||||
}
|
||||
def id = addMongoDB(o)
|
||||
o.remove("_id")
|
||||
o.id = id.toString()
|
||||
o.remove("create_time")
|
||||
addEs(o, "192.168.0.89", "development_zone")
|
||||
addEs(o, "192.168.0.52", "development_zone")
|
||||
d['I'] = "已完成"
|
||||
}else{
|
||||
println d['A'] +" 已存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addMongoDB(o){
|
||||
o.put("create_time", new Date())
|
||||
development_zone_100_CONN.save(o)
|
||||
DEVELOPMENT_ZONE_43_COLL.save(o)
|
||||
return o._id
|
||||
}
|
||||
|
||||
def addEs(map, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addDevAddress(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['D'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
q.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
q.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
q.put("park", d['C'])
|
||||
}
|
||||
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
|
||||
if(!mappingObj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", obj.province)
|
||||
o.put("city", obj.city)
|
||||
o.put("district", obj.district)
|
||||
o.put("name", obj.name)
|
||||
o.put("type", d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
o.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
o.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
o.put("park", d['C'])
|
||||
}
|
||||
def date = new Date()
|
||||
o.put("c_date", date)
|
||||
o.put("version", date.getTime())
|
||||
DEV_MAPPING_100_CONN.save(o)
|
||||
|
||||
|
||||
// o.id = o._id.toString()
|
||||
// o.remove("_id")
|
||||
// o.remove("c_date")
|
||||
// addEs(o, "192.168.0.89", "dev_mapping")
|
||||
// try{
|
||||
// addEs(o, "192.168.0.52", "dev_mapping")
|
||||
// }catch(Exception e){
|
||||
// println "52 es error: "+e.printStackTrace()
|
||||
// }
|
||||
|
||||
d['D'] = "已完成"
|
||||
}else{
|
||||
d['D'] = "重复"
|
||||
println d['A'] +" "+ d['C']+" 重复"
|
||||
}
|
||||
}else{
|
||||
d['D'] = "不存在"
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
def updateHasGov(province, city, district, street, hasGov){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", province)
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("street", street)
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
//修改43
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
|
||||
|
||||
//修改89 administrative_mapping
|
||||
update89EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
//修改89 administrative_mapping
|
||||
update52EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
}else{
|
||||
println province+" "+city+" "+district+" "+street+" 没找到"
|
||||
}
|
||||
//修改100 community
|
||||
//修改89 community
|
||||
//修改52 community
|
||||
|
||||
}
|
||||
|
||||
|
||||
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
|
||||
|
||||
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class DistrictAliasProcess {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictAliasProcess();
|
||||
// task.run()
|
||||
task.fenQiProcess()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d->
|
||||
if(d['H']=="是"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}else if(d['H']=="否"){
|
||||
def aliasDist = d['F']
|
||||
def arrs = aliasDist.split("-")
|
||||
def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district")
|
||||
if(obj){
|
||||
def alias = obj.alias
|
||||
def mapping_alias = obj.mapping_alias
|
||||
def udata = new BasicDBObject()
|
||||
if(alias && alias.contains(d['D'])){
|
||||
alias.remove(d['D'])
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
if(mapping_alias && mapping_alias.contains(d['D'])){
|
||||
mapping_alias.remove(d['D'])
|
||||
udata.put("mapping_alias", mapping_alias)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def fenQiProcess(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat","district")
|
||||
q.put("status", 0)
|
||||
q.put("master_district_name", 'master')
|
||||
def dbcursor = DbUtils.seaweed.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
|
||||
}
|
||||
|
||||
for(def r : rets){
|
||||
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
|
||||
def lastObj = null
|
||||
for(def j : rets){
|
||||
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
|
||||
lastObj = j;
|
||||
break
|
||||
}
|
||||
}
|
||||
if(lastObj){
|
||||
println r
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("master_district_name","")
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
+496
@@ -0,0 +1,496 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class GXDdistrictMatch {
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def regionCodeDataMap = PgsqlCommUtils.findDistrictMap();
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity();
|
||||
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
task.addOldMatch()
|
||||
// task.runDb()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
// task.runMatch()
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
//老匹配
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['AJ'] && d['I']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['AJ'] = jsonObject.get("id")
|
||||
d['AK'] = jsonObject.get("city")
|
||||
d['AL'] = jsonObject.get("district")
|
||||
d['AM'] = jsonObject.get("name")
|
||||
d['AN'] = jsonObject.get("address")
|
||||
|
||||
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||
if(dbo){
|
||||
d['AO'] = dbo.status
|
||||
d['AP'] = dbo.name_cat
|
||||
d['AQ'] = dbo.estate_type2
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def runDb(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(!d['U']){
|
||||
def fullCityObj = fullCityMap.get(d['AD'])
|
||||
if(fullCityObj){
|
||||
def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
|
||||
def obj = findPoiFromDB(fullCityObj.city, fullRegion, d['C'])
|
||||
if(obj){
|
||||
d['U'] = "地图库"
|
||||
d['V'] = obj.city
|
||||
d['W'] = obj.region
|
||||
d['X'] = obj.name
|
||||
d['Y'] = obj.type
|
||||
d['Z'] = obj.address
|
||||
d['AA'] = obj.lng
|
||||
d['AB'] = obj.lat
|
||||
}else{
|
||||
obj = findDatas(d['AD'], d['AE'], d['C'], d['C'].replaceAll('[·\\(\\)()]|·', ''))
|
||||
if(obj){
|
||||
d['U'] = "楼盘抓取库"
|
||||
d['V'] = obj.city
|
||||
d['W'] = obj.region
|
||||
d['X'] = obj.name
|
||||
d['Y'] = obj.property_type
|
||||
d['Z'] = obj.address
|
||||
def loc = obj.location
|
||||
if(loc){
|
||||
d['AA'] = loc.x
|
||||
d['AB'] = loc.y
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findPoiFromDB(city, region, name){
|
||||
def rets = EsSearchUtils.findPois(city, region, name)
|
||||
def obj = null
|
||||
if(rets && rets.size()==1){
|
||||
obj = rets[0]
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def fileName = "D:\\数据\\国信达\\接口匹配.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['V'] && d['A']){
|
||||
def city = getCityFromCityCode(d['A'])
|
||||
def region = regionCodeDataMap.get(d['A'] + "#" + d['E'])
|
||||
if (city && region) {
|
||||
region = RegionData.formatRegion(city, region)
|
||||
d['AD'] = city
|
||||
d['AE'] = region
|
||||
if (region) {
|
||||
d = match(city, region, d)
|
||||
} else {
|
||||
println d['A'] + "#" + d['E'] + " 区域无法格式化"
|
||||
}
|
||||
|
||||
} else {
|
||||
println d['A'] + "#" + d['E'] + " 无法格式化"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def runMatch(){
|
||||
def fileName = "D:\\数据\\福州\\建行测试匹配.xlsx"//11城重新匹配.xlsx 福州贝壳数据.xlsx
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['A']
|
||||
println d['C']
|
||||
if(d['B'] && d['A']){
|
||||
def region = RegionData.formatRegion(city, d['B'])
|
||||
if (region) {
|
||||
d = match(city, region, d)
|
||||
} else {
|
||||
println d['A'] + "#" + d['E'] + " 区域无法格式化"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateDistStatus(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
|
||||
def index = 0
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['U'] && ["名称匹配","名称匹配(Update)","名称在别名中","名称在别名中(Update)","别名匹配"].contains(d['U'])){
|
||||
index++
|
||||
|
||||
def status = d['Z']
|
||||
def name_cat = d['AA']
|
||||
|
||||
println d["V"]+" "+d["X"]+name_cat
|
||||
|
||||
if(status!=null && status.toString().toDouble().toInteger()==0){
|
||||
if(name_cat==null){//&& name_cat.toString().toDouble().toInteger()!=0
|
||||
|
||||
def dist = CommonDatautils.findDistrict(d["V"],d["W"],d["X"], "district")
|
||||
if(dist){
|
||||
def delUdata = new BasicDBObject()
|
||||
delUdata.put("name_cat", 0)
|
||||
delUdata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
|
||||
saveOptLog(dist.city, dist.region, dist.name, "update")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def restoreDist(){
|
||||
def fileName = "D:\\数据\\国信达\\11城小区需恢复数据.xlsx"//国信达11城(全).xlsx
|
||||
def index = 0
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['U']){
|
||||
index++
|
||||
def status = d['Z']
|
||||
println d["V"]+" "+d["X"]
|
||||
if(status && status.toString().toDouble().toInteger()==1){
|
||||
def dists = CommonDatautils.findAllStatusDistrict(d["V"],d["W"],d["X"], "district")
|
||||
if(dists && dists.size()==1){
|
||||
def dist = dists[0]
|
||||
def delUdata = new BasicDBObject()
|
||||
delUdata.put("name_cat", 0)
|
||||
delUdata.put("status", 0)
|
||||
delUdata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
|
||||
saveOptLog(dist.city, dist.region, dist.name, "update-status")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def saveToDB(){
|
||||
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['A']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def alias = []
|
||||
if(d['G'] && name!=d['G']){
|
||||
alias << d['G']
|
||||
}
|
||||
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
// alias << d['AI']
|
||||
// }
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
// if(d['AH']){
|
||||
// type = d['AH']
|
||||
// }else{
|
||||
// type = d['Y']
|
||||
// }
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
if(d['F']){
|
||||
Long lng = d['F']*10000000000
|
||||
udata.put("lng", lng)
|
||||
}
|
||||
if(d['E']){
|
||||
Long lat = d['E'] * 10000000000
|
||||
udata.put("lat", lat)
|
||||
}
|
||||
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
def opt = "add"
|
||||
if(s && s=='成功'){
|
||||
saveOptLog(udata.city, udata.region, udata.name, opt)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
def match(city, region, d){
|
||||
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
|
||||
if(!d['U']){
|
||||
d = matching(city, region, d, d['C'].replace("小区","").replace("号楼","号院").replace("号","号院"), d['D'], "(Update)")
|
||||
if(!d['U']){
|
||||
d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)")
|
||||
}
|
||||
|
||||
if(!d['U']){
|
||||
if(d['F'] && d['G']){
|
||||
d = matchAddr(city, region, d, "")
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['U']){
|
||||
d = matchApi(city, region, d, "property", false)
|
||||
if(!d['U']){
|
||||
d = matchApi(city, region, d, null, false)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['U']){
|
||||
d = matching2(city, region, d, d['C'], d['D'], "")
|
||||
}
|
||||
|
||||
if(!d['U']){
|
||||
if(d['C'].contains("小区")){
|
||||
d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)")
|
||||
}else{
|
||||
d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)")
|
||||
}
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
def matchApi(city, region, d, cat, useMap){
|
||||
def jsonObject = null
|
||||
if(!jsonObject && d['H']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
|
||||
}
|
||||
|
||||
if(!jsonObject && d['C']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
|
||||
if(!jsonObject && d['D']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
|
||||
}
|
||||
}
|
||||
if(jsonObject != null) {
|
||||
d['U'] = "接口匹配"
|
||||
d['V'] = jsonObject.get("city")
|
||||
d['W'] = jsonObject.get("district")
|
||||
d['X'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("category")//propertyType
|
||||
if(type && type!='null'){
|
||||
d['Y'] = type
|
||||
}
|
||||
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||
if(dbo){
|
||||
d['Z'] = dbo.status
|
||||
d['AA'] = dbo.name_cat
|
||||
}
|
||||
d['AC'] = jsonObject.get("credit")
|
||||
if(jsonObject.get("address") && jsonObject.get("address")!='null'){
|
||||
d['AH'] = jsonObject.get("address")
|
||||
}
|
||||
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
|
||||
def matching(city, region, d, name, alias, str){
|
||||
//正常状态下有没有
|
||||
def rets = CommonDatautils.findDistrictList(city, region, name, "district")
|
||||
if(rets){
|
||||
if(rets.size()>=1){
|
||||
d = setRowValue(rets[0], d, "名称匹配"+str)
|
||||
}
|
||||
}else{
|
||||
rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
if(rets && rets.size()>=1){
|
||||
d = setRowValue(rets[0], d, "名称在别名中"+str)
|
||||
}else{
|
||||
rets = CommonDatautils.findDistrictByAddress(city, region, name, "district")
|
||||
if(rets && rets.size()>=1){
|
||||
d = setRowValue(rets[0], d, "名称在地址中"+str)
|
||||
}else{
|
||||
if (d['D']) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
d = setRowValue(rets[0], d, "别名匹配"+str)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(rets){
|
||||
d['AI'] = rets.size()
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def matching2(city, region, d, name, alias, str){
|
||||
def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的
|
||||
if(rets){
|
||||
if (rets.size() >= 1) {
|
||||
d = setRowValue(rets[0], d, "名称匹配(非正常)"+str)
|
||||
}
|
||||
}else{
|
||||
|
||||
rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district")
|
||||
if (rets && rets.size() >= 1) {
|
||||
d = setRowValue(rets[0], d, "别名匹配(非正常)"+str)
|
||||
}else{
|
||||
rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district")
|
||||
if (rets.size() >= 1) {
|
||||
d = setRowValue(rets[0], d, "别名匹配(非正常)"+str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(rets){
|
||||
d['AI'] = rets.size()
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def matchAddr(city, region, d, str){
|
||||
def rets = CommonDatautils.findDistrictByAddress(city, region, d['F']+d['G'],"district")
|
||||
if(rets){
|
||||
if (rets.size() >= 1) {
|
||||
d = setRowValue(rets[0], d, "地址匹配")
|
||||
}
|
||||
}
|
||||
if(rets){
|
||||
d['AI'] = rets.size()
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
def setRowValue(dbo, d, remark){
|
||||
if (dbo) {
|
||||
d['V'] = dbo.city
|
||||
d['W'] = dbo.region
|
||||
d['X'] = dbo.name
|
||||
d['Y'] = dbo.estate_type2
|
||||
d['Z'] = dbo.status
|
||||
d['AA'] = dbo.name_cat
|
||||
d['U'] = remark
|
||||
d['AH'] = dbo.address
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def getCityFromCityCode(cityCode){
|
||||
def city = null
|
||||
for(def m : cityCodeDataMap){
|
||||
if(m.value == cityCode){
|
||||
city = m.key
|
||||
break;
|
||||
}
|
||||
}
|
||||
return city;
|
||||
}
|
||||
|
||||
def filterSpacialChar(def s){
|
||||
def pattern = ~/[()!;\·]\[【】()]/
|
||||
def matcher = pattern.matcher(s)
|
||||
|
||||
def count = matcher.getCount()
|
||||
if(count && count>0){
|
||||
return matcher.replaceAll('_')
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def findDatas(def city, def region, def name1, def name2){
|
||||
def sql = "select * from community where city='${city}' and region='${region}' and (name='${name1}' or name='${name2}')"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
println d.property_type
|
||||
})
|
||||
|
||||
if(rets){
|
||||
def obj = null
|
||||
for(def r : rets){
|
||||
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
|
||||
obj = r
|
||||
break
|
||||
}
|
||||
}
|
||||
if(obj){
|
||||
return obj;
|
||||
}else{
|
||||
return rets[0];
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, type){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category","district");
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", type)
|
||||
obj.put("user_name", "高娟")
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
}
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 名录集
|
||||
*/
|
||||
class MinglujiData {
|
||||
|
||||
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new MinglujiData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def prov = "浙江省"
|
||||
def city = "嘉兴"
|
||||
// exportDatas(city)
|
||||
matchDev(prov, city)
|
||||
matchCommunity(city)
|
||||
|
||||
// tempMatchDev(prov)
|
||||
// saveMljData()
|
||||
|
||||
fiveLevelSplit(city)
|
||||
}
|
||||
def exportDatas(city){
|
||||
def datas = findMinglujiDatas(city)
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
|
||||
}
|
||||
def matchDev(prov, city){
|
||||
def devs = loadAllDevs(prov)
|
||||
|
||||
def i = 1;
|
||||
while(i<20) {
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if (d['D']) {
|
||||
def street = d['D']
|
||||
def splitStreet = ""
|
||||
if (street) {
|
||||
if (street.contains(d['C'])) {
|
||||
def start = street.toString().lastIndexOf(d['C']) + d['C'].size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
} else {
|
||||
splitStreet = street
|
||||
}
|
||||
}
|
||||
d['K'] = splitStreet
|
||||
}
|
||||
|
||||
if (d['I']) {
|
||||
def splitCommunity = ""
|
||||
if (d['C'] && d['I'].toString().contains(d['C'])) {
|
||||
splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size())
|
||||
} else {
|
||||
splitCommunity = d['I']
|
||||
}
|
||||
d['L'] = splitCommunity
|
||||
}
|
||||
|
||||
|
||||
if (d['E'] && !d['N']) {
|
||||
def obj = findDevByName(d['E'])
|
||||
if (!obj) {
|
||||
obj = findDevByAlias(d['E'])
|
||||
if (!obj) {
|
||||
obj = findDebBySimlar(devs, d['E'], d['B'])
|
||||
if (!obj) {
|
||||
obj = findDevByPark(d['A'], d['B'], d['E'])
|
||||
if (!obj) {
|
||||
} else {
|
||||
d['N'] = "园区"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "相似"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "别名"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "相同"
|
||||
}
|
||||
if (obj) {
|
||||
d['M'] = obj.name
|
||||
d['O'] = obj.city
|
||||
d['P'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
i++
|
||||
println fileName +" 完成(匹配开发区不使用接口)"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def tempMatchDev(prov){
|
||||
def devs = loadAllDevs(prov)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d->
|
||||
if(d['U'] && !d['W'] ){
|
||||
def obj = findDevByName(d['U'])
|
||||
if(!obj){
|
||||
obj = findDevByAlias(d['U'])
|
||||
if(!obj){
|
||||
obj = findDebBySimlar(devs, d['U'], d['B'])
|
||||
if(!obj){
|
||||
obj = findDevByPark(d['A'], d['B'], d['U'])
|
||||
if (!obj) {
|
||||
}else{
|
||||
d['X'] = "园区"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "相似"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "别名"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "相同"
|
||||
}
|
||||
if(obj){
|
||||
d['W'] = obj.name
|
||||
d['Y'] = obj.city
|
||||
d['Z'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchCommunity(city){
|
||||
def i = 1;
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['A'] && d['B'] && d['L'] && !d['Q']){
|
||||
def list = findCommunityByName(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
|
||||
}else{
|
||||
d['T'] = "包含"
|
||||
}
|
||||
}else{
|
||||
d['T'] = "相同"
|
||||
}
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
def obj = list[0]
|
||||
d['Q'] = obj.name
|
||||
d['R'] = obj.street_name
|
||||
d['U'] = obj.area_name
|
||||
d['V'] = obj.city_name
|
||||
}else if(list.size()>=2){
|
||||
for(def l : list){
|
||||
if(d['C']==l.area_name){
|
||||
d['Q'] = l.name
|
||||
d['R'] = l.street_name
|
||||
d['U'] = l.area_name
|
||||
d['V'] = l.city_name
|
||||
break;
|
||||
}
|
||||
}
|
||||
d['S'] = list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
println fileName +" 运行完成"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def fiveLevelSplit(city){
|
||||
def i = 1;
|
||||
while(i<20) {
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['J'] && !d['X']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['J'])
|
||||
if(jsonObject) {
|
||||
if (jsonObject.get("address") && jsonObject.get("address") != "null") {
|
||||
d['X'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
def saveMljData(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("province", d['A'])
|
||||
obj.put("city", d['B'])
|
||||
obj.put("district", d['C'])
|
||||
obj.put("name", d['D'])
|
||||
obj.put("address", d['E'])
|
||||
obj.put("dev_name", d['F'])
|
||||
obj.put("committee", d['G'])
|
||||
obj.put("township", d['H'])
|
||||
company_100_CONN.save(obj)
|
||||
})
|
||||
}
|
||||
def loadAllDevs(prov){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", prov)
|
||||
return development_zone_CONN.find(q).toList()
|
||||
}
|
||||
|
||||
def findDebBySimlar(def devs, def name, def city){
|
||||
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
def last = null
|
||||
for(def d : devs){
|
||||
def alias1 = null
|
||||
if(!d.alias){
|
||||
alias1 = []
|
||||
}else{
|
||||
alias1 = d.alias
|
||||
}
|
||||
if(!alias1.contains(d.name)){
|
||||
alias1 << d.name
|
||||
}
|
||||
if(alias1){
|
||||
try {
|
||||
for(def n : alias1){
|
||||
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
|
||||
last = d
|
||||
break
|
||||
}
|
||||
}
|
||||
}catch(exception){
|
||||
println alias1
|
||||
}
|
||||
|
||||
}
|
||||
if(last){
|
||||
break
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
def findDevByName(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByAlias(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('alias', new BasicDBObject('$in',[name]))
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevBySimlar(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByPark(prov, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("type", "园区")
|
||||
return dev_mapping_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findCommunityByName(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", name)
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findCommunityBySimlar(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findMinglujiDatas(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("c_address",1)
|
||||
keys.put("c_city",1)
|
||||
keys.put("c_committee",1)
|
||||
keys.put("c_community",1)
|
||||
keys.put("c_company",1)
|
||||
keys.put("c_district",1)
|
||||
keys.put("c_street",1)
|
||||
keys.put("kind",1)
|
||||
keys.put("name",1)
|
||||
keys.put("province",1)
|
||||
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
|
||||
def rets = []
|
||||
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
|
||||
for(def d : datas){
|
||||
def street = d.c_street
|
||||
def splitStreet = ""
|
||||
if(street){
|
||||
if(street.contains(d.c_district)){
|
||||
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
}
|
||||
}
|
||||
def splitCommunity = ""
|
||||
if(d.c_committee){
|
||||
if(d.c_committee.contains(d.c_district)){
|
||||
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
|
||||
}
|
||||
}
|
||||
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+423
@@ -0,0 +1,423 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
|
||||
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
|
||||
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
// task.run();
|
||||
task.regionAndStreetProcess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = [""]
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
def regionAndStreetProcess(){
|
||||
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
|
||||
for(def city : citys){
|
||||
def datas = findAllPoisByCity(city)
|
||||
process(datas)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllPoisByCity(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findPois(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>=10000){
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
// updateToEs2(datas)
|
||||
// updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
// updateToEs2(datas)
|
||||
// updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def process(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
dataRun(d)
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def checkData(def index, def datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = running(d)
|
||||
rets << map
|
||||
if(map.update_status && map.update_status!=100){
|
||||
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
|
||||
}
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
updateToMG(map)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def regions = getFullRegion(d)
|
||||
def town = null
|
||||
def map =[:]
|
||||
if(d.location){
|
||||
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if(d.district){
|
||||
for(def rs : regions){
|
||||
if(rs.shape && rs.shape.coordinates){
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(regionFlag){
|
||||
//判断township是否在统计局中
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, d.city, d.township)
|
||||
}
|
||||
if(town){
|
||||
if(town.street == d.township){
|
||||
//打标记
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = town.street
|
||||
}
|
||||
}else{
|
||||
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
|
||||
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
|
||||
if (mapTown) {
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = mapTown.street
|
||||
}else{
|
||||
if(d.district!=gaodeMap.district){
|
||||
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
//打标记“经纬度有误,不在区域围栏中”
|
||||
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
def dataRun(d){
|
||||
def regions = getFullRegion(d)
|
||||
if(d.location) {
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if (d.district) {
|
||||
for (def rs : regions) {
|
||||
if (rs.shape && rs.shape.coordinates) {
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (regionFlag) {//在围栏里
|
||||
//判断township是否在统计局中
|
||||
// if(d.township){
|
||||
// town = getSimilarTown(regions, d.city, d.township)
|
||||
// }
|
||||
|
||||
|
||||
}else{//不在转栏里,重新搜索
|
||||
|
||||
println(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getFullRegion(d){
|
||||
def regions = regionMap.get(d.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.district)){
|
||||
names << r
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
def getSimilarTown(regions, city, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(city+"#"+r.district)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id, def township){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.township='${township}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def updateToEs2(rows) {
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.update_status && map.update_status==2){
|
||||
map.remove("run_status")
|
||||
map.remove("update_status")
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def updateToMG(m){
|
||||
// def rets = []
|
||||
// for(def i=0;i<rows.size();i++){
|
||||
// def m = rows[i]
|
||||
def udata = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
if(k!="id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", m.id)
|
||||
if(udata){
|
||||
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println m.toString()+" 无法修改-无值"
|
||||
}
|
||||
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class TempExportData {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new TempExportData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
|
||||
for (def city : citys) {
|
||||
def cityRets = getResults(city)
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def getResults(city) {
|
||||
def rets = []
|
||||
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for (def i = 0; i < dists.size(); i++) {
|
||||
def d = dists[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.addAndGet(1)
|
||||
try {
|
||||
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
|
||||
if (objs) {
|
||||
def ret = []
|
||||
ret << d._id.toString()
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
ret << d.address
|
||||
|
||||
def str = ""
|
||||
for (def o : objs) {
|
||||
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
|
||||
}
|
||||
ret << str
|
||||
ret << objs.size()
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
println(city +" "+rets.size())
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class Test{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
def task = new Test()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def i=1
|
||||
while(i<2){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
def rets = process(districts)
|
||||
println i + " "+districts.size() +" "+rets.size()
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def process(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
methodA(rMap)
|
||||
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
|
||||
rets << rMap.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def methodA(rMap) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
// 模拟一个可能会抛出异常的操作
|
||||
riskyOperation();
|
||||
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
|
||||
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(10000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
|
||||
}
|
||||
}
|
||||
|
||||
// 一个可能会抛出异常的操作
|
||||
def riskyOperation() throws Exception {
|
||||
// 模拟一个有50%概率抛出异常的操作
|
||||
if (Math.random() < 0.5) {
|
||||
throw new Exception("Simulated exception");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThardParthAnalyBatch extends BaseBatch{
|
||||
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThardParthAnalyBatch(), args)
|
||||
}
|
||||
|
||||
// static main(args) {
|
||||
// def task = new ThardParthAnalyBatch()
|
||||
// task.run()
|
||||
// }
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def fjwTotalCount = CommonDatautils.findAllCount("district")
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
println "统计月份:"+sDate+" 查询更新时间:"+mdate
|
||||
|
||||
def crawlTotalCount = findAllEsDataCount(mdate)
|
||||
println "抓取第三方小区总量:"+crawlTotalCount
|
||||
def synchTotalCount = findAllMongoDatasCount(sDate)
|
||||
println "同步到第三方库小区总量: "+synchTotalCount
|
||||
|
||||
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
|
||||
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
|
||||
|
||||
def synchDistCount = findAllMongoDistrictCount(sDate)
|
||||
println "同步到第三方库有价格小区量: "+synchDistCount
|
||||
|
||||
def matchTotalCount = findMatchSuccessCount(sDate)
|
||||
println "第三方匹配成功总量:"+matchTotalCount
|
||||
|
||||
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
|
||||
println "第三方匹配成功小区量:"+matchDistTotalCount
|
||||
|
||||
def precent = matchDistTotalCount*100.0/fjwTotalCount
|
||||
precent = DataHandleUtils.getDoubleRound(2, precent)
|
||||
println "匹配成功占比:"+precent+"%"
|
||||
|
||||
def noMatchTotalCount = findNoMatchCount(sDate)
|
||||
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
|
||||
|
||||
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findAllEsDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllEsDistDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllMongoDatasCount(s_date){
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findAllMongoDistrictCount(s_date){
|
||||
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("region",1)
|
||||
keys.put("name",1)
|
||||
def dbcursor = thirdPartyColl.find(q,keys)
|
||||
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findAllMongoDatasHavePricesCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessDistCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findNoMatchCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
|
||||
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fjw_dist_total", fjwTotalCount)
|
||||
udata.put("crawl_total", crawlTotalCount)
|
||||
udata.put("synch_total", synchTotalCount)
|
||||
udata.put("synch_have_price_total", synchHavePriceTotalCount)
|
||||
udata.put("synch_have_price_district_total", synchDistCount)
|
||||
udata.put("match_total", matchTotalCount)
|
||||
udata.put("match_dist_total", matchDistTotalCount)
|
||||
udata.put("match_dist_rate", precent)
|
||||
udata.put("no_match_total", noMatchTotalCount)
|
||||
udata.put("c_date", new Date())
|
||||
if(obj){
|
||||
udata.put("m_date", new Date())
|
||||
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
udata.put("s_date", sDate)
|
||||
THIRD_PARTY_STAT.save(udata)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+1167
File diff suppressed because it is too large
Load Diff
+51
@@ -0,0 +1,51 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class TownshipPrcoess {
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipPrcoess();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run() {
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇别名问题.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(!d['G']){
|
||||
// def name1 = d['E'].replace("街道","").replace("镇","").replace("乡","")
|
||||
// def name2 = d['F'].replace("街道","").replace("镇","").replace("乡","")
|
||||
// if(name1 == name2){
|
||||
// d['G'] = "不处理"
|
||||
// }
|
||||
|
||||
def lists1 = findCommunityTown(community_CONN, d['C'], d['D'], d['E'])
|
||||
def lists2 = findCommunityTown(community_CONN, d['C'], d['D'], d['F'])
|
||||
if(lists1 && lists2){
|
||||
d['G'] = "需删除"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findCommunityTown(community_CONN, city, region, town){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", city)
|
||||
q.put("area_name", region)
|
||||
q.put("street_name", town)
|
||||
def rets = community_CONN.find(q).toList()
|
||||
return rets;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+632
@@ -0,0 +1,632 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class TownshipStrandUpdate {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
def fullCityMap = null
|
||||
def regionMap = null
|
||||
def commMap = null
|
||||
def polylineMap = null
|
||||
def newAdvMappingMap = null
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
// task.process()
|
||||
// task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
// task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
// task.updateDevMapping()
|
||||
println new Date().getTime()
|
||||
// task.updateSpecCityTown()
|
||||
}
|
||||
|
||||
|
||||
void process() {
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
polylineMap = CommonDatautils.findAllTownRanges()
|
||||
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
|
||||
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
|
||||
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
//
|
||||
// if(["中山","东莞"].contains(city)){
|
||||
// continue
|
||||
// }
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
// if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
// 相同且为统计局乡镇
|
||||
// 更正
|
||||
// 不同且不为统计局乡镇
|
||||
// 区域不同
|
||||
// 地图无乡镇街道
|
||||
// 城市不同
|
||||
|
||||
def stat(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
def rets = []
|
||||
rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"]
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
def map = [:]
|
||||
def datas = null
|
||||
if(file.exists()){
|
||||
datas = ExcelUtils.read(fileName,0, 1)
|
||||
for(def d : datas){
|
||||
def key = "空值"
|
||||
if(d['S']){
|
||||
key = d['S']
|
||||
}
|
||||
def count = map.get(key)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(key, count+1)
|
||||
}
|
||||
}
|
||||
|
||||
println city
|
||||
map.each {k,v->
|
||||
println k+" "+v
|
||||
}
|
||||
println "--------------------------------------------------------------------------------------------"
|
||||
if(map){
|
||||
rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")]
|
||||
}
|
||||
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺")
|
||||
}
|
||||
|
||||
def updateTown(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
println fileName
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['S']!='相同且为统计局乡镇' && d['S']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['B'])
|
||||
q.put("name", d['C'])
|
||||
q.put("status", 0)
|
||||
q.put("cat","shop")
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", d['R'])
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def checkData(def dists){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets << running(d)
|
||||
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return dists
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
|
||||
regions = []
|
||||
regions << fullCity.city
|
||||
}
|
||||
def gaodeMap = null
|
||||
def town = null
|
||||
def mapTown = null
|
||||
def oldSimlarTown = null
|
||||
def newSimlarTown = null
|
||||
if(d.lng){
|
||||
d.lng = d.lng/10000000000.0
|
||||
d.lat = d.lat/10000000000.0
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, fullCity, d.township)
|
||||
}
|
||||
|
||||
if(town && town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
d.polygon_town = town.street
|
||||
}
|
||||
}
|
||||
d.strand_street = ""
|
||||
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
|
||||
if(mapTown){
|
||||
d.strand_street = mapTown.street
|
||||
if(mapTown.shape && mapTown.shape.coordinates){
|
||||
d.map_town_shape = "有"
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
|
||||
if(flag){
|
||||
d.map_polygon_town = mapTown.street
|
||||
}
|
||||
}
|
||||
}else{
|
||||
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
|
||||
if(oldSimlarTown){
|
||||
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
|
||||
d.old_similar_town_score = oldSimlarTown.score
|
||||
}
|
||||
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
|
||||
if(newSimlarTown){
|
||||
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
|
||||
d.new_similar_town_score = newSimlarTown.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def mapCity = null
|
||||
def mapRegion = null
|
||||
def mapTownship = null
|
||||
def mapAddr = null
|
||||
if(gaodeMap){
|
||||
if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
mapCity = gaodeMap.city
|
||||
}
|
||||
if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
mapRegion = gaodeMap.district
|
||||
}else{
|
||||
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
|
||||
mapRegion = mapCity
|
||||
}
|
||||
}
|
||||
if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTownship = gaodeMap.township
|
||||
}
|
||||
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
mapAddr = gaodeMap.format_addr
|
||||
}
|
||||
}
|
||||
d.mapCity = mapCity
|
||||
d.mapRegion = mapRegion
|
||||
d.mapTownship = mapTownship
|
||||
d.mapAddr = mapAddr
|
||||
if(mapCity && mapRegion){
|
||||
if(mapCity == fullCity.city){
|
||||
if(regions.contains(mapRegion)){
|
||||
if(mapTownship){
|
||||
if(town){
|
||||
d.old_tjj_township = "是"
|
||||
}else{
|
||||
d.old_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTown){
|
||||
d.new_tjj_township = "是"
|
||||
}else{
|
||||
d.new_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTownship == d.township){
|
||||
if(town){
|
||||
d.remark = "相同且为统计局乡镇"
|
||||
}else{
|
||||
d.remark = "非统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}else{
|
||||
if(mapTown){
|
||||
d.remark = "更正"
|
||||
}else{
|
||||
d.remark = "不同且不为统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d.remark = "地图无乡镇街道"
|
||||
d.opt = "地图无街道信息"
|
||||
}
|
||||
|
||||
}else{
|
||||
d.remark = "区域不同"
|
||||
d.opt = "经纬度可能跨区域"
|
||||
}
|
||||
}else{
|
||||
d.remark = "城市不同"
|
||||
d.opt = "经纬度可能跨城市"
|
||||
}
|
||||
}
|
||||
|
||||
if(d.map_town_shape && d.map_town_shape=="有"){
|
||||
if(!d.map_polygon_town){
|
||||
d.opt = "经纬度可疑"
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def getSimilarTownByStr(fullCity, townstr, townList){
|
||||
|
||||
def maxScore = 0
|
||||
def town = null
|
||||
for(def townObj : townList){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
|
||||
if(score > maxScore){
|
||||
maxScore = score
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
town.score = maxScore
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getSimilarTown(regions, fullCity, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(fullCity.city+"#"+r)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getFullRegion(fullCity, d){
|
||||
def regions = regionMap.get(fullCity.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
|
||||
names << r.district
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
|
||||
def createDatas(dists){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
|
||||
for(def d : dists){
|
||||
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(township || district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateDiffRegionTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
|
||||
d['AA'] = lists.size()
|
||||
//只处理库中只有一个的
|
||||
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
|
||||
def newRegion = RegionData.formatRegion(d['B'], d['M'])
|
||||
if(newRegion){
|
||||
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
def dbo = lists[0]
|
||||
def arrs = d['X'].split(" ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", arrs[1])
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
|
||||
d['AB'] = "已处理"
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def cleanTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['AC']){
|
||||
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
|
||||
if(dbo && dbo.township){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkTownFromMongo(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
citys = citys.reverse()
|
||||
def index = 0
|
||||
for(def city : citys) {
|
||||
index++
|
||||
if (["中山", "东莞"].contains(city)) {
|
||||
continue
|
||||
}
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
|
||||
for (def d : dists) {
|
||||
if (d.township) {
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
def town = getSimilarTown(regions, fullCity, d.township)
|
||||
if (!town) {
|
||||
town = findStreet(d.city, d.city, d.township)
|
||||
if (!town) {
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
|
||||
|
||||
println d.city + " " + d.region + " " + d.township + " " + d.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println index + " " + city + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def findStreet(city, district, street){
|
||||
def q = new BasicDBObject()
|
||||
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("city", new BasicDBObject('$regex', provPattern))
|
||||
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
return list[0]
|
||||
}else{
|
||||
println city+ " "+district+" "+street+" 有多个"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def updateTownMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['F']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mzb_district_name", d['F'])
|
||||
//
|
||||
// def updateQuery = new BasicDBObject()
|
||||
// updateQuery.put("city", d['C'])
|
||||
// updateQuery.put("district", d['D'])
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
|
||||
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
if(city){
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(region){
|
||||
def updateQuery = new BasicDBObject()
|
||||
updateQuery.put("category", "region")
|
||||
updateQuery.put("city", city)
|
||||
updateQuery.put("name", region)
|
||||
|
||||
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
|
||||
if(list && list.size()==1){
|
||||
def dbo = list[0]
|
||||
def query = new BasicDBObject()
|
||||
query.put("_id", dbo._id)
|
||||
|
||||
// def pattern = dbo.pattern
|
||||
// if(pattern.contains(d['F'])){
|
||||
// pattern.remove(d['F'])
|
||||
// }
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("org_name", d['F'])
|
||||
// udata.put("pattern",pattern)
|
||||
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
|
||||
}else{
|
||||
println city +" "+region+" "+list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateDevMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("district", d['B'])
|
||||
q.put("name",d['C'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("township", d['D'])
|
||||
def list = dev_MAPPING_COLL_100.find(q).toList()
|
||||
if(list && list.size()==2){
|
||||
def obj = list[0]
|
||||
def removeQuery = new BasicDBObject()
|
||||
removeQuery.put("_id", obj._id)
|
||||
dev_MAPPING_COLL_100.remove(removeQuery)
|
||||
}else{
|
||||
println q
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateSpecCityTown(){
|
||||
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
|
||||
|
||||
// def citys = ["中山"]
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
|
||||
if(list.size()>1){
|
||||
println d['A'] +" "+d['C'] + " 有多个"
|
||||
}else if(list.size()==1){
|
||||
def dbo = list[0]
|
||||
def town = null
|
||||
// if(d['X']){
|
||||
// town = d['X']
|
||||
// }else
|
||||
if(d['R']){
|
||||
town = d['R']
|
||||
}
|
||||
if(!town && d['M']){
|
||||
town = d['M']
|
||||
}
|
||||
def region = RegionData.formatRegion(d['A'], town)
|
||||
if(region){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", region)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", town)
|
||||
udata.put("region", region)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class TradeDatasDistBatch {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new TradeDatasDistBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
udata.put("name", d['C'])
|
||||
udata.put("alias", d['D'])
|
||||
alias_fang_COLL.save(udata)
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def flag = null
|
||||
// if(!dist){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// flag = "别名匹配"
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }else{
|
||||
// flag = "名称匹配"
|
||||
// }
|
||||
// if(flag){
|
||||
// d['F'] = dist.city
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
})
|
||||
}
|
||||
}
|
||||
+41
-11
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.addrepo.InvokeHttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
@@ -135,6 +136,26 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def format(def city, def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
if(address){
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
@@ -228,9 +249,9 @@ class FangjiaClientUtils {
|
||||
def url = null;
|
||||
try {
|
||||
address = address.replace("{","").replace("\\","")
|
||||
url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
|
||||
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
@@ -314,13 +335,13 @@ class FangjiaClientUtils {
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
@@ -332,18 +353,27 @@ class FangjiaClientUtils {
|
||||
public static def formatNinelevel(city, address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
|
||||
def url = null
|
||||
address = address.replace("{","")
|
||||
int retryCount = 0
|
||||
while (retryCount < 3) {
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
if (content) {
|
||||
break
|
||||
}
|
||||
retryCount++
|
||||
}
|
||||
if (!content) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
println address +" "+content+" "+e.getMessage()
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
+5
-7
@@ -36,18 +36,16 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
||||
}
|
||||
|
||||
|
||||
if(!rMap && address){
|
||||
if(!rMap && address && name){
|
||||
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true, name)
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false, null)
|
||||
}
|
||||
// if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
||||
// if (obj) {
|
||||
|
||||
+8
-1
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
@@ -16,7 +17,13 @@ class EsSelectTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new EsSelectTest()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.test()
|
||||
}
|
||||
|
||||
|
||||
def test(){
|
||||
def ret = CommonDatautils.findLogs("1007741", "/property/evaluate", "2024-03-01", "2024-03-28")
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 房价网内部模型估值项目
|
||||
* 小区建筑类型数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataCommbuildtypeBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rets = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
for(def city : citys){
|
||||
def distIds = FjwPgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
|
||||
|
||||
def rows = createDatas(cityCodeDatas.get(city), distIds)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rets)
|
||||
}
|
||||
|
||||
def createDatas(cityCode, dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def id = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def dist = CommonDatautils.findDistrictFieldById(id, ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
|
||||
if(dist){
|
||||
def ret = createData(cityCode, dist)
|
||||
if(ret){
|
||||
rets.addAll(ret)
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(cityCode, dist){
|
||||
|
||||
def types = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
def rets = []
|
||||
for(def type : types){
|
||||
def ret = []
|
||||
ret << cityCode
|
||||
ret << dist._id.toString()
|
||||
ret << type
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
|
||||
rets << ret
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.bigdata_commbuildtype")
|
||||
println "base.bigdata_commbuildtype 数据已删除"
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
def sqls = []
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
def sql = "insert into base.bigdata_commbuildtype values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
println "start: "+new Date()
|
||||
def index =0
|
||||
sqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
index++
|
||||
println index
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
|
||||
}
|
||||
}
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 房价网内部估值项目使用
|
||||
* 小区数据同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
def fangjiaDists = new HashSet()
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
|
||||
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
|
||||
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
}
|
||||
|
||||
// if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
// }
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
|
||||
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
if(!fangjiaDists.contains(ccbId)){
|
||||
def datas = ccbMaps.get(ccbId)
|
||||
datas[datas.size()-1] = 1
|
||||
rets << datas
|
||||
}
|
||||
}
|
||||
println "添加删除状态小区:"+rets.size()
|
||||
return rets
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
|
||||
//def rets = Collections.synchronizedList(new ArrayList());
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDataMap, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(def cityCodeDataMap, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
try{
|
||||
def data = distCodeMap.get(dist.region)
|
||||
if(!data){
|
||||
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret << cityCodeDataMap.get(dist.city)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist._id.toString()
|
||||
if(data){
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
def lat = dist.lat/10000000000d
|
||||
def lng = dist.lng/10000000000d
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "pa101"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.street){
|
||||
ret << dist.street
|
||||
if(dist.street_number){
|
||||
ret << dist.street_number
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
// def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
||||
// if(jsonObj && jsonObj.result && jsonObj.result.street){
|
||||
// if(jsonObj.result.street){
|
||||
// ret << jsonObj.result.street
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
//
|
||||
// if(jsonObj.result.streetNumber){
|
||||
// ret << jsonObj.result.streetNumber
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
// }else{
|
||||
// ret << ""
|
||||
// ret << ""
|
||||
// }
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << 0
|
||||
}catch(Exception e){
|
||||
println dist.city + " "+dist.name+" error"
|
||||
e.printStackTrace()
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public def findPgCcbDistrict(cityid){
|
||||
def sql = "select * from base.bigdata_community where urbn_id='${cityid}'";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.comm_nm
|
||||
data << r.gujia_dtl_adr
|
||||
data << r.gujia_bds_ltt
|
||||
data << r.gujia_bds_lgt
|
||||
data << r.gujiabdsocpoldtotarea
|
||||
data << r.gujiabds_bld_tot_area
|
||||
data << r.gujia_bds_grnrate
|
||||
data << r.gujia_bds_vol_rate
|
||||
data << r.gujia_bds_park_num
|
||||
data << r.gujia_bds_bld_yr_rc
|
||||
data << r.gujia_bds_hs_use_val
|
||||
data << r.totunits
|
||||
data << r.gujia_bds_data_tm_val
|
||||
data << r.gujiabds_udt_flag_val
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_comm_tp_val
|
||||
data << r.gujia_comm_fmnm
|
||||
data << r.gujia_str
|
||||
data << r.gujia_hsno
|
||||
data << r.gujiawthr_new_pan_ind
|
||||
data << r.gujia_comm_intd
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.gujia_src_co
|
||||
data << r.gujia_vlgtwn_str_mng
|
||||
data << r.gujia_lgc_admndiv_cd
|
||||
data << r.gujia_lgc_adiv
|
||||
data << r.gujia_bds_bds_st
|
||||
map.put(r.gujia_comm_id, data)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.bigdata_community")
|
||||
println "base.bigdata_community 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_community values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
def saveToPgSql2(rows){
|
||||
def newRows = []
|
||||
for(def r : rows) {
|
||||
def newR = []
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
// r[i] = null
|
||||
newR << null
|
||||
}else{
|
||||
newR << r[i]
|
||||
}
|
||||
}
|
||||
newR << null
|
||||
newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
newR << null
|
||||
newR << '2999-12-31'
|
||||
newR << null
|
||||
newR << null
|
||||
newR << null
|
||||
|
||||
newRows << newR
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
def index = 0
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
index++
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
|
||||
index
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("\\|@\\|")
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
def updateVersion(def time){
|
||||
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
|
||||
for(def tableName : tables){
|
||||
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+790
@@ -0,0 +1,790 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.*
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 成交数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataDealBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
println "历史id加载量:"+dealIds.size()
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
println "txt 生成 " + rows.size()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
// saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("\\|@\\|")
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
dealIds = Collections.synchronizedList(dealIds);
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
// def lianjiaPriceMap = findThirdPartyPrice("上海")
|
||||
// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
|
||||
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
||||
|
||||
def ccbPriceMap = null
|
||||
if(city == "上海"){
|
||||
ccbPriceMap = [:]//getCommunityPrices("上海")
|
||||
}
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
if(dealIds.contains(trade._id.toString())){
|
||||
println "重复:"+trade._id.toString() + " "+ trade.city+ " " +trade.district_name
|
||||
}else{
|
||||
if(source == "youda"){
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
}
|
||||
}else{
|
||||
def ret = converData(trade, cityCodeDatas, distCodeMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await();
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getCommunityPrices(def city){
|
||||
def sql = "select commid, avg_price from community_base_price where year_month=${endDate} and cityname='${city}'";
|
||||
def map = [:]
|
||||
try{
|
||||
sqlInstance.eachRow(sql,{
|
||||
map.put(it.commid, it.avg_price)
|
||||
});
|
||||
}catch(Exception e){
|
||||
//getPpeGTE15Datas(city)
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for (def db : dbcursor) {
|
||||
def price = null
|
||||
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
|
||||
price = db.lianjia_esf_price
|
||||
}
|
||||
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
|
||||
price = db.beike_esf_price
|
||||
}
|
||||
if(price){
|
||||
map.put(db.region+"##"+db.name, price)
|
||||
}
|
||||
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def ccbPriceMap){
|
||||
def ret = []
|
||||
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
def price = ccbPriceMap.get(obj._id.toString())
|
||||
if(price){
|
||||
def mod = java.lang.Math.abs((trade.price-price)*100/price)
|
||||
if(mod<=20){
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
def data = distCodeMap.get(trade.fjRegion)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}
|
||||
if(!street){
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.match_region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
}
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << CcbCommonUtils.getCcbCity(trade.city)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.dist_id.toString()
|
||||
ret << trade.master_name
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
|
||||
ret << ""//期数
|
||||
ret << trade.floor
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.avg_price
|
||||
ret << trade.total_price
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
def root = sourceMap.get(trade.source)
|
||||
if(root){
|
||||
if(CcbCommonUtils.rootMap.get(root)){
|
||||
ret << CcbCommonUtils.rootMap.get(root)
|
||||
}else{
|
||||
ret << "99"
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.total_price
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){
|
||||
def housetype = null
|
||||
if(buildtype && ["0112","0110","0109"].contains(buildtype)){
|
||||
housetype = "07"
|
||||
}else if(buildtype && ["0107","0106","0105"].contains(buildtype)){
|
||||
housetype = "16"
|
||||
}else{
|
||||
if(roomcnt && roomcnt==4){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "14"
|
||||
}else{
|
||||
housetype = "11"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==3){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "13"
|
||||
}else{
|
||||
housetype = "10"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==2){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "12"
|
||||
}else{
|
||||
housetype = "09"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==1){
|
||||
housetype = "08"
|
||||
}else{
|
||||
housetype="99"
|
||||
}
|
||||
}
|
||||
return housetype;
|
||||
}
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all")
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def price = statMap.get(dist.region+"@"+dist.name)
|
||||
def startPrice = null
|
||||
def endPrice = null
|
||||
// if(price){
|
||||
// if(dist.city=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else if(dist.city=="杭州"){
|
||||
// startPrice = (int)(price - price*0.15)
|
||||
// endPrice = (int)(price*0.15 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// }
|
||||
// def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
|
||||
// if(!trades && dist.address){
|
||||
// def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
// if(flag){
|
||||
// trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
|
||||
//
|
||||
// if(!trades){
|
||||
// trades = CcbCommonUtils.findTradesDetailByAddressAlias(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def trades = CcbCommonUtils.findTradesDetailById(dist, startDate, endDate)
|
||||
if(trades){
|
||||
// def unitPrice= ppeDistsMap.get(dist._id.toString())
|
||||
// if(unitPrice){
|
||||
// for(def t : trades){
|
||||
// t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
|
||||
// }
|
||||
// }
|
||||
tradesList.addAll(trades)
|
||||
}
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return tradesList
|
||||
}
|
||||
|
||||
def getYouDaData(statMap){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
// q.put("status", 0)
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def obj : dbcursor){
|
||||
|
||||
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
|
||||
rets << obj;
|
||||
// def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
|
||||
// if(price){
|
||||
// def startPrice = 0
|
||||
// def endPrice= 0
|
||||
// if(obj.fjCity=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// if(obj.price>=startPrice && obj.price<=endPrice){
|
||||
// rets << obj;
|
||||
// }else{
|
||||
// index++
|
||||
// println index + " "+obj
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", new BasicDBObject('$gte', weekend))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
def price = map.get(d.region+"##"+d.district_name)
|
||||
if(!price && d.value_type==2){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findHistoryDealIds(def city){
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.bigdata_deal where urbn_nm='${city}'"
|
||||
def ids = []
|
||||
try{
|
||||
pgsqlInstance.eachRow(sql,{
|
||||
ids << it.gujia_bds_mdl_id
|
||||
});
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd")
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(trades.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for(def i=0;i<trades.size();i++){
|
||||
final def t = trades[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.addAndGet(1)
|
||||
try{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
if(t.house_use && t.house_use.contains("商住")){
|
||||
if(!t.name.contains("商住")){
|
||||
t.name = t.name +"商住楼"
|
||||
}
|
||||
}
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
}
|
||||
}
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
println count_.get()+" "+t+" "+ rMap
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def delPgData(){
|
||||
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
pgsqlInstance.execute("delete from base.bigdata_deal where p9_start_date='${month}'")
|
||||
println "base.bigdata_deal 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_deal values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
|
||||
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
|
||||
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql2(rows){
|
||||
def newRows = []
|
||||
for(def r : rows) {
|
||||
def newR = []
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
// r[i] = null
|
||||
newR << null
|
||||
}else{
|
||||
newR << r[i]
|
||||
}
|
||||
}
|
||||
newR << null
|
||||
newR << "2023-09-28"
|
||||
newR << null
|
||||
newR << '2999-12-31'
|
||||
newR << null
|
||||
newR << null
|
||||
newR << null
|
||||
|
||||
newRows << newR
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
|
||||
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
|
||||
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
+969
@@ -0,0 +1,969 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.google.common.hash.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
|
||||
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
|
||||
|
||||
|
||||
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
|
||||
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataForsaleBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//目前查询库中是否已存在,用的是Oracle
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def startDate = lastMonth+"18"
|
||||
def endDate = curMonth+"19" //正式运行时curMonth要改
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = null;//findAllHouseIdsByCity()
|
||||
|
||||
for(def city : citys){
|
||||
index++
|
||||
houseIdsfilter = findAllHouseIdsByCity(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def statCity(def citys){
|
||||
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
|
||||
def houseCitys = []
|
||||
for(def d : datas){
|
||||
houseCitys << d['C']
|
||||
}
|
||||
for(def city : citys){
|
||||
if(!houseCitys.contains(city)){
|
||||
def scity = CcbCommonUtils.specialCityMap.get(city)
|
||||
if(!scity){
|
||||
println city
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
|
||||
|
||||
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
|
||||
println city + "(原)房源总量:"+retMap.size()
|
||||
|
||||
def allDatas = retMap.values()
|
||||
saveToPgSql(allDatas)
|
||||
}
|
||||
|
||||
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
|
||||
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def commId = rMap.get("_id").toString()
|
||||
|
||||
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap)
|
||||
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
|
||||
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
|
||||
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||
// }
|
||||
if(ret){
|
||||
def areaData = distCodeMap.get(rMap.region)
|
||||
if(areaData){
|
||||
def regionCode = areaData.district_code
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
|
||||
m.city_code = cityCode
|
||||
m.region = areaData.district
|
||||
m.region_code = regionCode
|
||||
m.district_id = commId
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
curhousefilter.put(m.houseId)
|
||||
houseIdsfilter.put(m.houseId)
|
||||
}else{
|
||||
println city +" "+m.houseId+" 已存在"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+rMap.region+" 没有找到对应的区域"
|
||||
}
|
||||
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
|
||||
|
||||
} catch (Exception e) {
|
||||
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def converData(def m){
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
data << m.region
|
||||
data << m.district_id
|
||||
data << m.district_name
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.address==null?"":m.address==null
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << ""
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.room_structure_code==null?"":m.room_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
data << m.hall==null?"":m.hall
|
||||
data << ""
|
||||
data << m.toilet==null?"":m.toilet
|
||||
data << m.direction_code==null?"":m.direction_code
|
||||
data << m.fitment_code==null?"":m.fitment_code
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.p_date==null?"":m.p_date
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.root
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.c_date==null?"":m.c_date
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
data << ""
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
|
||||
|
||||
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
// if(rets1){
|
||||
// def newRets = []
|
||||
// def cityCode = cityCodeDataMap.get(city)
|
||||
// def houseIds = getAllHouseIds(commId, cityCode)
|
||||
// for(def r : rets1){
|
||||
// if(!houseIds.contains(r.houseId)){
|
||||
// newRets << r
|
||||
// }else{
|
||||
// println r.houseId +" 已存在"
|
||||
// }
|
||||
// }
|
||||
// return newRets
|
||||
// }
|
||||
// return [];
|
||||
return rets1;
|
||||
}
|
||||
|
||||
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
// if(seawater.height && seawater.height<1){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
|
||||
// if(houseIds.contains(seawater.id)){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
// def count = isExistFromHouse(seawater.id)
|
||||
// if(count){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
println name +" "+houses.size()+" last:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
|
||||
//用以下方法会查出发布时间很久以前的
|
||||
// QueryBuilder dateQueryBuilder = QueryBuilders.boolQuery()
|
||||
// dateQueryBuilder.should(date_range)
|
||||
// dateQueryBuilder.should(date_range_mdate)
|
||||
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
// .must(area_range).must(dateQueryBuilder)
|
||||
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
|
||||
def ids = new HashSet()
|
||||
for(def h : rets1){
|
||||
ids << h.houseId
|
||||
}
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
if(ids && ids.contains(seawater.id)){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
// if(seawater.height && seawater.height<1){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
|
||||
// if(houseIds.contains(seawater.id)){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
// def count = isExistFromHouse(seawater.id)
|
||||
// if(count){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
println name +" "+houses.size()+" last:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
def rets = []
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(3000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public boolean isVilla(area, name, height, room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String csvHandlerStr(String str) {
|
||||
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
String tempDescription=str;
|
||||
//如果有逗号
|
||||
if(str.contains(",")||str.contains(",")){
|
||||
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if(str.contains("\"")){
|
||||
// tempDescription=str.replace("\"", "\"\"");
|
||||
tempDescription=str.replace("\"", "");
|
||||
}
|
||||
//在将逗号转义
|
||||
tempDescription="\""+tempDescription+"\"";
|
||||
}
|
||||
return tempDescription;
|
||||
}
|
||||
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
def getAllHouseIds(def commid, def cityCode){
|
||||
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'";
|
||||
|
||||
def houseIds = new HashSet()
|
||||
sqlInstance.eachRow(sql,{it->
|
||||
houseIds << it.FORSALEID
|
||||
});
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
def findAllHouseIdsByCity(cityid){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
// def houseIds = new HashSet()
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_id='${cityid}'",{it->
|
||||
filter.put(it.gujia_sell_id)
|
||||
});
|
||||
|
||||
// pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale_2021",{it->
|
||||
// filter.put(it.gujia_sell_id)
|
||||
// });
|
||||
return filter;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_forsale values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40],
|
||||
r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58])
|
||||
ps.toString()
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
}
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import org.bson.types.ObjectId
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 人工指导价同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
|
||||
|
||||
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwGuiDingPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
|
||||
def priceIdsfilter = FjwPgsqlCommUtils.findGuiDingIdsByCity()
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
|
||||
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
|
||||
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
}
|
||||
println "【房价网内部】人工指导价总计:"+rows.size()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def findFangDatas(def districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println " 小区量:"+districts.size()
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findMatchThirdPrice(dist)
|
||||
if(obj && obj.avg_price){
|
||||
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return result
|
||||
}
|
||||
|
||||
def findPersonDatas(def districts, def dataMap){
|
||||
def result = []
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
def dist = districts[i];
|
||||
def price = dataMap.get(dist._id.toString())
|
||||
if (price) {
|
||||
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
|
||||
println dist.city + " " + dist.region + " " + dist.name + " " + price
|
||||
}
|
||||
}
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
def findMatchThirdPrice(dist){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", dist.city)
|
||||
q.put("match_region", dist.region)
|
||||
q.put("match_name", dist.name)
|
||||
q.put("s_date", sDate)
|
||||
q.put("source","搜房")
|
||||
def obj = dataProcessColl.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(priceIdsfilter.mightContain(ps._id.toString())){
|
||||
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
|
||||
continue
|
||||
}
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
if(areaData){
|
||||
ret << ps._id.toString()
|
||||
ret << dist._id.toString()
|
||||
ret << dist.name
|
||||
ret << cityCodeDatas.get(dist.city)
|
||||
ret << CcbCommonUtils.getCcbCity(dist.city)
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.guiding_value
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createDatas(cityCodeDatas, distCodeMap, datas){
|
||||
def rets = []
|
||||
if(datas) {
|
||||
for (def ps : datas) {
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(ps.region)
|
||||
if (areaData) {
|
||||
ret << ps.row_id
|
||||
ret << ps.dist_id
|
||||
ret << ps.name
|
||||
ret << cityCodeDatas.get(ps.city)
|
||||
ret << ps.city
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.avg_price
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
|
||||
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], "all");
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.guided_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(300, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
/**
|
||||
* 房价网内部使用
|
||||
*/
|
||||
class FjwPgsqlCommUtils {
|
||||
|
||||
public static def database = "fangjia"
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
public static def findAllDistrictIdsByOracle(def cityid){
|
||||
def sql = "select gujia_comm_id from base.bigdata_community where urbn_id='${cityid}' and gujia_bds_bds_st=0";
|
||||
def set = new HashSet()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
set << r.gujia_comm_id
|
||||
});
|
||||
return set
|
||||
}
|
||||
|
||||
public static def findDistGovPrice(def city){
|
||||
def ids = []
|
||||
sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it->
|
||||
ids << it.gujia_bds_id
|
||||
});
|
||||
return ids;
|
||||
}
|
||||
|
||||
|
||||
public static def deleteGuidedPrice(def dealdate){
|
||||
def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'"
|
||||
sqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
public static def findGuiDingIdsByCity(){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
sqlInstance.eachRow("SELECT gujia_fsid FROM base.guided_price",{it->
|
||||
filter.put(it.gujia_fsid)
|
||||
});
|
||||
return filter
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+553
@@ -0,0 +1,553 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 第三方均价数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304//
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwThirdPartyPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
runDatas(citys)
|
||||
|
||||
//同步历史数据
|
||||
// def stop = false
|
||||
// def month = -1
|
||||
// while(!stop){
|
||||
// if(sDate>202404 && sDate<202407){
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
// println weekend.toString() +" "+sDate
|
||||
// runDatas(citys)
|
||||
// }
|
||||
// month = month -1
|
||||
// sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
// if(sDate == 201912){
|
||||
// stop = true;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
def runDatas(citys){
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
|
||||
def gocIdsPrices = []
|
||||
if(city == "深圳"){
|
||||
gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city)
|
||||
}
|
||||
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all")
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
|
||||
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices)
|
||||
rows.addAll(rets)
|
||||
println index+" "+city+" "+rets.size() +" "+rows.size()
|
||||
}
|
||||
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def ret = []
|
||||
ret << third.id.toString()
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
ret << cityCodeDatas.get(d.city)
|
||||
ret << d.city
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
ret << sDate
|
||||
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
ret << third.zhugezhaofang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
ret << third.anjuke_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
ret << third.soufang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.lianjia_esf_price){
|
||||
ret << third.lianjia_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.cityhouse_esf_price){
|
||||
ret << third.cityhouse_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.maitian_esf_price){
|
||||
ret << third.maitian_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.beike_esf_price){
|
||||
ret << third.beike_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.get("365house_esf_price")){
|
||||
ret << third.get("365house_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.zhongyuan_esf_price){
|
||||
ret << third.zhongyuan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.tuboshi_esf_price){
|
||||
ret << third.tuboshi_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
ret << third.qfang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangtan_esf_price){
|
||||
ret << third.fangtan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangduoduo_esf_price){
|
||||
ret << third.fangduoduo_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
ret << third.get("5i5j_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){
|
||||
def allRows = []
|
||||
for(def d : dists){
|
||||
if(gocPrices && gocPrices.contains(d._id.toString())){
|
||||
continue;
|
||||
}
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def dataMap = [:]
|
||||
dataMap.COMMID = d._id.toString()
|
||||
dataMap.COMMNAME = d.name
|
||||
dataMap.CITYID = cityCodeDatas.get(d.city)
|
||||
dataMap.CITYNAME = d.city
|
||||
dataMap.DISTID = data.district_code
|
||||
dataMap.DISTNAME = data.district
|
||||
dataMap.YEAR_MONTH = sDate
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
price_map.zhuge_avg_price = third.zhugezhaofang_esf_price
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
price_map.anjuke_avg_price = third.anjuke_esf_price
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
price_map.fang_avg_price = third.soufang_esf_price
|
||||
}
|
||||
if(third.lianjia_esf_price){
|
||||
price_map.lianjia_avg_price = third.lianjia_esf_price
|
||||
}
|
||||
if(third.cityhouse_esf_price){
|
||||
price_map.cityhouse_avg_price = third.cityhouse_esf_price
|
||||
}
|
||||
if(third.maitian_esf_price){
|
||||
price_map.maitian_avg_price = third.maitian_esf_price
|
||||
}
|
||||
if(third.beike_esf_price){
|
||||
price_map.ke_avg_price = third.beike_esf_price
|
||||
}
|
||||
if(third.get("365house_esf_price")){
|
||||
price_map.taofang_avg_price = third.get("365house_esf_price")
|
||||
}
|
||||
if(third.zhongyuan_esf_price){
|
||||
price_map.zhongyuan_avg_price = third.zhongyuan_esf_price
|
||||
}
|
||||
if(third.tuboshi_esf_price){
|
||||
price_map.tbs_avg_price = third.tuboshi_esf_price
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
price_map.qfang_avg_price = third.qfang_esf_price
|
||||
}
|
||||
if(third.fangtan_esf_price){
|
||||
price_map.fangtan_avg_price = third.fangtan_esf_price
|
||||
}
|
||||
if(third.fangduoduo_esf_price){
|
||||
price_map.fangduoduo_avg_price = third.fangduoduo_esf_price
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "房产超市"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
|
||||
def rets = []
|
||||
for (def d : dists) {
|
||||
|
||||
def third = thirdPriceMap.get(d.region + "##" + d.name)
|
||||
if (third) {
|
||||
def data = distCodeMap.get(d.region)
|
||||
if (data) {
|
||||
third.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << cityCodeDatas.get(d.city)
|
||||
ret << data.district_code
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
def web = ""
|
||||
if(k=="zhugezhaofang_esf_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_esf_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="soufang_esf_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_esf_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_esf_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_esf_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="beike_esf_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="365house_esf_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_esf_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tuboshi_esf_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_esf_price"){
|
||||
ret << "Q房"
|
||||
}else if(k=="fangtan_esf_price"){
|
||||
ret << "房探"
|
||||
}else if(k=="fangduoduo_esf_price"){
|
||||
ret << "房多多"
|
||||
}else if(k=="5i5j_esf_price"){
|
||||
ret << "我爱我家"
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << sDate
|
||||
ret << v
|
||||
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
dataMap.put("id", db._id.toString())
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
dataMap.put("soufang_esf_price", db.soufang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
dataMap.put("qfang_esf_price", db.qfang_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
def resultMap = [:]
|
||||
map.each {key, value->
|
||||
if(value && value.size()>1){
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
|
||||
def price = statMap.get(key)
|
||||
if(price){
|
||||
def priceMap = [:]
|
||||
def startPrice = (int)(price - price*0.3)
|
||||
def endPrice = (int)(price*0.3 + price)
|
||||
value.each{pk,pv->
|
||||
if(pk.contains("price")){
|
||||
if(pv>=startPrice && pv<=endPrice){
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}else{
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}
|
||||
if(priceMap.size()>1){
|
||||
resultMap.put(key, priceMap)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
resultMap.put(key, value)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return resultMap
|
||||
}
|
||||
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+31
-33
@@ -13,7 +13,7 @@ class MsbPocTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
@@ -24,7 +24,7 @@ class MsbPocTest {
|
||||
// cityPrice()
|
||||
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
|
||||
|
||||
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
|
||||
addCCBPrice(inputfile)
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
@@ -69,21 +69,21 @@ class MsbPocTest {
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['S'] && row['D']){
|
||||
if(!row['Y'] && row['G']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['H']
|
||||
def region = null //row['D']
|
||||
def name = row['E']
|
||||
def address = row['G']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['S'] = jsonObject.get("id")
|
||||
row['T'] = jsonObject.get("city")
|
||||
row['U'] = jsonObject.get("district")
|
||||
row['V'] = jsonObject.get("name")
|
||||
row['W'] = jsonObject.get("propertyType")
|
||||
row['X'] = jsonObject.get("credit")
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
row['AC'] = jsonObject.get("propertyType")
|
||||
// row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -111,23 +111,18 @@ class MsbPocTest {
|
||||
|
||||
def match(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
|
||||
if(!row['Y']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def region = null //row['B']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
row['J'] = jsonObject.get("id")
|
||||
row['K'] = jsonObject.get("city")
|
||||
row['L'] = jsonObject.get("district")
|
||||
row['M'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['N'] = type
|
||||
}
|
||||
row['O'] = jsonObject.get("credit")
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
if(jsonObject != null) {
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -174,19 +169,22 @@ class MsbPocTest {
|
||||
def addCCBPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['M']){
|
||||
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
if(dist){
|
||||
def d = CommonDatautils.findDistrictById(dist._id.toString())
|
||||
if(!row['V']){
|
||||
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
// if(dist){
|
||||
if(row['Y'] instanceof String){
|
||||
def d = CommonDatautils.findDistrictById(row['Y'])
|
||||
if(d){
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
|
||||
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
|
||||
if(map){
|
||||
row['M'] = map.avg_price
|
||||
row['V'] = map.avg_price
|
||||
// row['I'] = map.calc_type
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+56
-31
@@ -33,17 +33,31 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "C:\\Users\\gaojuan\\Desktop\\7Ô³ɽ»°¸Àýbadcase.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
exportPkPrice()
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
// addTradePrice(filePath)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addTradePrice(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
if(d['Y']){
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
|
||||
if (tradePrice) {
|
||||
d['W'] = tradePrice.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
@@ -155,13 +169,13 @@ class AddPriceTest {
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['M']){
|
||||
def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['M'], cityId)
|
||||
if(d['U']){
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(map){
|
||||
d['AK'] = map.avg_price
|
||||
d['AL'] = map.calc_type
|
||||
d['AM'] = map.credit_value
|
||||
d['F'] = map.avg_price
|
||||
// d['P'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -169,7 +183,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -190,11 +204,11 @@ class AddPriceTest {
|
||||
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['M'])
|
||||
if (!d['X']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['AN'] = thirdPrice
|
||||
d['X'] = thirdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -206,8 +220,8 @@ class AddPriceTest {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
|
||||
|
||||
q.put("s_date", new BasicDBObject('$gte', 202403))
|
||||
// q.put("s_date", 202404)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
for(def db : dbcursor){
|
||||
@@ -216,10 +230,10 @@ class AddPriceTest {
|
||||
break;
|
||||
}
|
||||
|
||||
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
|
||||
price = db.beike_esf_price
|
||||
break
|
||||
}
|
||||
// if(!price && db.beike_esf_price && db.beike_esf_price>2000){
|
||||
// price = db.beike_esf_price
|
||||
// break
|
||||
// }
|
||||
}
|
||||
return price
|
||||
}
|
||||
@@ -227,12 +241,12 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['B']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if (!d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['L'] = gxdPrice
|
||||
d['M'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -256,17 +270,28 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(dist){
|
||||
// if(!d['L']){
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['AJ'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
d['E'] = dist.name
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.name_cat
|
||||
// d['F'] = dist.estate_type2
|
||||
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
|
||||
// d['H'] = dist.master_district_name
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
d['K'] = dbo.avg_price_2b
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['G'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
// d['E'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// }
|
||||
// }
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||
import groovy.json.JsonOutput
|
||||
import net.sf.json.JSONObject
|
||||
import okhttp3.*;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Proxy;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
||||
import okhttp3.Request.Builder;
|
||||
|
||||
class QiliangTest {
|
||||
String proxyHost = 'tunnel2.qg.net';
|
||||
Integer proxyPort = 14186;
|
||||
String proxyUsername = '113BBC06';
|
||||
String proxyPassword = '3B2FBD05B0AF';
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
|
||||
static main(args) {
|
||||
QiliangTest task = new QiliangTest();
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
print(s);
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\对公地址(问题案例)跑一下结果.xlsx", { row ->
|
||||
if(row['B']){
|
||||
def parmas = [
|
||||
'address': row['B'],
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def str = getProxyResult(parmas);
|
||||
println str
|
||||
JSONObject result = JSONObject.fromObject(str)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
if(data.province && data.province!='null'){
|
||||
row['C'] = data.province
|
||||
}
|
||||
if(data.provinceCode && data.provinceCode!='null'){
|
||||
row['D'] = data.provinceCode
|
||||
}
|
||||
if(data.city && data.city!='null'){
|
||||
row['E'] = data.city
|
||||
}
|
||||
if(data.cityCode && data.cityCode!='null'){
|
||||
row['F'] = data.cityCode
|
||||
}
|
||||
|
||||
if(data.district && data.district!='null'){
|
||||
row['G'] = data.district
|
||||
}
|
||||
if(data.districtCode && data.districtCode!='null'){
|
||||
row['H'] = data.districtCode
|
||||
}
|
||||
|
||||
if(data.town && data.town!='null'){
|
||||
row['I'] = data.town
|
||||
}
|
||||
if(data.townCode && data.townCode!='null'){
|
||||
row['J'] = data.townCode
|
||||
}
|
||||
|
||||
if(data.village && data.village!='null'){
|
||||
row['K'] = data.village
|
||||
}
|
||||
if(data.villageCode && data.villageCode!='null'){
|
||||
row['L'] = data.villageCode
|
||||
}
|
||||
|
||||
if(data.areaCode && data.areaCode!='null'){
|
||||
row['M'] = data.areaCode
|
||||
}
|
||||
|
||||
|
||||
if(data.farmer == "0"){
|
||||
row['N'] = "非农"
|
||||
}else{
|
||||
row['N'] = "农"
|
||||
}
|
||||
if(data.isGovt == "0"){
|
||||
row['O'] = "否"
|
||||
}else{
|
||||
row['O'] = "是"
|
||||
}
|
||||
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
row['P'] = s
|
||||
}
|
||||
if(data.govtAddr && data.govtAddr!='null'){
|
||||
row['Q'] = data.govtAddr?data.govtAddr:''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
String getProxyResult(data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
String jsonData = new JsonOutput().toJson(data);
|
||||
// 创建代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
|
||||
// 创建 URL 对象
|
||||
URL url = new URL(targetUrl);
|
||||
|
||||
// 创建连接对象
|
||||
URLConnection connection = url.openConnection(proxy);
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST");
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true);
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json");
|
||||
|
||||
// 发送请求参数
|
||||
OutputStream outputStream = connection.getOutputStream();
|
||||
outputStream.write(jsonData.getBytes("UTF-8"));
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 读取响应
|
||||
int responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line);
|
||||
}
|
||||
reader.close();
|
||||
return response.toString();
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
+5
@@ -200,4 +200,9 @@ public class SignatureUtils {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
def a = getAliAddressStrc("上海市长宁区新泾三村")
|
||||
println a
|
||||
}
|
||||
}
|
||||
|
||||
+66
-68
@@ -18,77 +18,75 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
|
||||
|
||||
//def s = getTownshipInfo("118.702988,39.740979")
|
||||
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
//// if(!row['I']){
|
||||
//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"ÈËÃñÕþ¸®")
|
||||
//// if(rets){
|
||||
//// def obj = findOneByName(rets, row['C']+"ÈËÃñÕþ¸®")
|
||||
//// if(obj){
|
||||
//// row['I'] = obj.lng
|
||||
//// row['J'] = obj.lat
|
||||
//// println row['C'] + " "+obj
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(row['D'] && !row['F']){
|
||||
//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
//// def map = ["lon":row['D'], "lat":row['E']]
|
||||
//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
|
||||
//// row['F'] = township
|
||||
//// }
|
||||
//
|
||||
//// if(row['F'] && !row['J']){
|
||||
//// def q = new BasicDBObject()
|
||||
//// q.put("city_name", row['B'])
|
||||
//// q.put("area_name", row['C'])
|
||||
//// def streetName = row['F']
|
||||
//// q.put("street_name", streetName)
|
||||
//// def count = COMMUNITY_CONN.count(q);
|
||||
//// row['J'] = count
|
||||
////// row['H'] = streetName
|
||||
//// }
|
||||
//// if(!row['H']){
|
||||
//// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
|
||||
//// if(gaodeMap && gaodeMap.township){
|
||||
//// row['H'] = gaodeMap.township
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(row['J'] && row['J'].toString().toDouble()>0){
|
||||
//// def up_query = new BasicDBObject()
|
||||
//// up_query.put("city_name", row['B'])
|
||||
//// up_query.put("area_name", row['C'])
|
||||
//// up_query.put("street_name", row['F'])
|
||||
//// def udata = new BasicDBObject()
|
||||
//// udata.put("has_gov", "ÊÇ")
|
||||
//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
|
||||
//// }
|
||||
//
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
for(def d : objs){
|
||||
if(!d.name_pinyin){
|
||||
if(d.name){
|
||||
println d._id
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_pinyin", py1)
|
||||
if(d.location){
|
||||
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
|
||||
udata.put("location", ["lat":lat, "lon":lon])
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if(!row['I'] && row['I']=='宝应县'){
|
||||
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||
if(rets){
|
||||
def obj = findOneByName(rets, row['C']+"人民政府")
|
||||
if(obj){
|
||||
row['I'] = obj.lng
|
||||
row['J'] = obj.lat
|
||||
println row['C'] + " "+obj
|
||||
}
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(row['D'] && !row['F']){
|
||||
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
def map = ["lon":row['D'], "lat":row['E']]
|
||||
def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
|
||||
row['F'] = township
|
||||
}
|
||||
|
||||
// if(row['F'] && !row['J']){
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city_name", row['B'])
|
||||
// q.put("area_name", row['C'])
|
||||
// def streetName = row['F']
|
||||
// q.put("street_name", streetName)
|
||||
// def count = COMMUNITY_CONN.count(q);
|
||||
// row['J'] = count
|
||||
//// row['H'] = streetName
|
||||
// }
|
||||
// if(!row['H']){
|
||||
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// row['H'] = gaodeMap.township
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(row['J'] && row['J'].toString().toDouble()>0){
|
||||
// def up_query = new BasicDBObject()
|
||||
// up_query.put("city_name", row['B'])
|
||||
// up_query.put("area_name", row['C'])
|
||||
// up_query.put("street_name", row['F'])
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("has_gov", "是")
|
||||
// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
|
||||
// }
|
||||
})
|
||||
|
||||
|
||||
//def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
//for(def d : objs){
|
||||
// if(!d.name_pinyin){
|
||||
// if(d.name){
|
||||
// println d._id
|
||||
// def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_pinyin", py1)
|
||||
// if(d.location){
|
||||
// def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
|
||||
// def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
|
||||
// udata.put("location", ["lat":lat, "lon":lon])
|
||||
// }
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", d._id)
|
||||
// COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||
import groovy.json.JsonOutput
|
||||
import net.sf.json.JSONObject
|
||||
import okhttp3.*;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Proxy;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
||||
import okhttp3.Request.Builder;
|
||||
|
||||
class QiliangTest1 {
|
||||
def proxyHost = 'tunnel2.qg.net'
|
||||
def proxyPort = 14186
|
||||
def proxyUsername = '113BBC06'
|
||||
def proxyPassword = '3B2FBD05B0AF'
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
|
||||
static main(args) {
|
||||
def task = new QiliangTest()
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
|
||||
'type': '1'
|
||||
]
|
||||
def s = task.getProxyResult(data)
|
||||
print(s)
|
||||
}
|
||||
|
||||
def run(){//地址测试-2024.3.4-结果.xlsx
|
||||
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
|
||||
if(!row['N']){
|
||||
def address = null
|
||||
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
|
||||
address = row['B']+row['C']
|
||||
}else{
|
||||
address = row['C']
|
||||
}
|
||||
if(address){
|
||||
def result = getHttpResult(address)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
if(data.province && data.province!='null'){
|
||||
row['N'] = data.province
|
||||
}
|
||||
if(data.city && data.city!='null'){
|
||||
row['O'] = data.city
|
||||
}
|
||||
if(data.district && data.district!='null'){
|
||||
row['P'] = data.district
|
||||
}
|
||||
if(data.town && data.town!='null'){
|
||||
row['Q'] = data.town
|
||||
}
|
||||
if(data.village && data.village!='null'){
|
||||
row['R'] = data.village
|
||||
}
|
||||
|
||||
if(data.areaCode && data.areaCode!='null'){
|
||||
row['S'] = data.areaCode
|
||||
}
|
||||
|
||||
if(data.farmer == "0"){
|
||||
row['T'] = "非农"
|
||||
}else{
|
||||
row['T'] = "农"
|
||||
}
|
||||
if(data.isGovt == "0"){
|
||||
row['U'] = "否"
|
||||
}else{
|
||||
row['U'] = "是"
|
||||
}
|
||||
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
row['V'] = s
|
||||
}
|
||||
if(data.govtAddr && data.govtAddr!='null'){
|
||||
row['W'] = data.govtAddr?data.govtAddr:''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
JSONObject getHttpResult(address){
|
||||
JSONObject data = new JSONObject()
|
||||
data.put("address", address)
|
||||
data.put("type","2")
|
||||
|
||||
Map headers = new HashMap();
|
||||
headers.put("Accept", "application/json, text/plain, */*");
|
||||
headers.put("Accept-Encoding", "gzip, deflate");
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
headers.put("Content-Type","application/json;charset=UTF-8")
|
||||
headers.put("Host","146.56.217.44:9010")
|
||||
headers.put("Origin","http://146.56.217.44:8088")
|
||||
headers.put("Proxy-Connection","keep-alive")
|
||||
headers.put("Referer","http://146.56.217.44:8088/")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
||||
|
||||
HttpRequest request1 = new HttpRequest()
|
||||
try{
|
||||
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
|
||||
JSONObject response = JSONObject.fromObject(res)
|
||||
println response
|
||||
return response
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def getProxyResult(data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
def jsonData = new JsonOutput().toJson(data)
|
||||
// 创建代理对象
|
||||
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
|
||||
|
||||
// 创建 URL 对象
|
||||
def url = new URL(targetUrl)
|
||||
|
||||
// 创建连接对象
|
||||
def connection = url.openConnection(proxy)
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST")
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true)
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json")
|
||||
|
||||
// 发送请求参数
|
||||
def outputStream = connection.getOutputStream()
|
||||
outputStream.write(jsonData.getBytes("UTF-8"))
|
||||
outputStream.flush()
|
||||
outputStream.close()
|
||||
|
||||
// 读取响应
|
||||
def responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
|
||||
def response = new StringBuilder()
|
||||
String line
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line)
|
||||
}
|
||||
reader.close()
|
||||
return response.toString()
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}"
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticUtils89 {
|
||||
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
// connectionProperties.setProperty("user","developer");
|
||||
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
|
||||
connectionProperties.setProperty("user","elastic");
|
||||
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
if(results.size()%100000==0){
|
||||
println "."+ new Date()
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
}
|
||||
+436
-13
File diff suppressed because one or more lines are too long
+17
-16
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xingyebank.Data
|
||||
@@ -34,16 +35,18 @@ class PoiWash {
|
||||
public static def regions = []
|
||||
public static def nameKeys = null
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
// task.updateId()
|
||||
// task.run()
|
||||
// task.synch()
|
||||
// task.check()
|
||||
task.deduplicate()
|
||||
// task.deduplicate()
|
||||
// task.updateEsMapPoi()
|
||||
// task.checkSpecialSymbols()
|
||||
// task.splitRoad()
|
||||
task.splitRoad()
|
||||
}
|
||||
|
||||
def updateId(){
|
||||
@@ -452,9 +455,10 @@ class PoiWash {
|
||||
|
||||
def splitRoad(){
|
||||
//"ÈýÃ÷ÊÐ",
|
||||
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys) {
|
||||
def datas = findPoiByQuery(city, ['city','address'])
|
||||
def cityObj = fullCityMap.get(city)
|
||||
def datas = findPoiByQuery(cityObj.city, ['city','address'])
|
||||
splitRoadRun(datas)
|
||||
println city +" Íê³É"
|
||||
}
|
||||
@@ -472,18 +476,15 @@ class PoiWash {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
|
||||
if(jsonObject && jsonObject.containsKey("result")){
|
||||
def dataMap = jsonObject.get("result")
|
||||
if(dataMap.get("street")){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street", dataMap.get("street"))
|
||||
if(dataMap.get("streetNumber")){
|
||||
udata.put("street_number", dataMap.get("streetNumber"))
|
||||
}
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
|
||||
udata.put("m_date", new Date())
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
|
||||
} catch (Exception e) {
|
||||
|
||||
+46
-22
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -26,8 +28,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
|
||||
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
|
||||
@@ -40,7 +42,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
@@ -61,11 +63,13 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
//codeRun()
|
||||
// codeRun()
|
||||
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
|
||||
// addTest()
|
||||
// updateTownship()
|
||||
}
|
||||
|
||||
def addTest(){
|
||||
@@ -125,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
def run(){
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
// priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
@@ -346,24 +350,44 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
def codeRun(){
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// def dbcursor = mca_administrative_division_coll.find(q)
|
||||
// def rets = []
|
||||
// for(def it : dbcursor){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("_id", it._id)
|
||||
// udata.put("province", it.province)
|
||||
// udata.put("city", it.city)
|
||||
// udata.put("district", it.district)
|
||||
// udata.put("mca_province_code", it.province_code)
|
||||
// udata.put("mca_city_code", it.city_code)
|
||||
// udata.put("mca_district_code", it.district_code)
|
||||
// udata.put("level", it.level)
|
||||
// udata.put("c_date", it.c_date)
|
||||
// rets << udata
|
||||
//// administrative_division_100_CONN.save(udata)
|
||||
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
// }
|
||||
// administrative_division_100_CONN.insert(rets)
|
||||
//
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = mca_administrative_division_coll.find(q)
|
||||
def rets = []
|
||||
for(def it : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("_id", it._id)
|
||||
udata.put("province", it.province)
|
||||
udata.put("city", it.city)
|
||||
udata.put("district", it.district)
|
||||
udata.put("mca_province_code", it.province_code)
|
||||
udata.put("mca_city_code", it.city_code)
|
||||
udata.put("mca_district_code", it.district_code)
|
||||
udata.put("level", it.level)
|
||||
udata.put("c_date", it.c_date)
|
||||
rets << udata
|
||||
// administrative_division_100_CONN.save(udata)
|
||||
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
q.put("province","澳门特别行政区")
|
||||
def dbcursor = administrative_division_2022_CONN.find(q)
|
||||
for(def d : dbcursor){
|
||||
administrative_division_100_CONN.save(d)
|
||||
|
||||
}
|
||||
administrative_division_100_CONN.insert(rets)
|
||||
}
|
||||
|
||||
def updateTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
|
||||
if(row['Q'] && row['Q']=='是'){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", row['J'])
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.hubingliang
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import groovy.sql.Sql
|
||||
|
||||
@@ -23,7 +24,7 @@ import com.mongodb.DBObject
|
||||
@BatchOwner(Watcher.Hubingliang)
|
||||
class ApiInvokeStatistics extends BaseBatch{
|
||||
final static def mongoDomain = "mongo.db.fangjia.com"
|
||||
public static final DBCollection OPEN_URI_LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com", 27020)
|
||||
public static final DBCollection OPEN_URI_LOG = com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI,"business", "open_uri_log")
|
||||
public static final DBCollection OPEN_URI_LOG_COUNT = MongoUtils.getCollectionByName("business", "api_invoke_statistical", "mongo.db.fangjia.com", 27017)
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
static main(args) {
|
||||
|
||||
+18
-1
@@ -98,6 +98,23 @@ class InsidePolygon {
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<List<Point>> createPoint4(def pointList){
|
||||
List<List<Point>> alllist = new ArrayList<>();
|
||||
for(def pointArr : pointList){
|
||||
List<Point> list = new ArrayList<>();
|
||||
for(def arr : pointArr){
|
||||
Point point = new Point();
|
||||
double x = arr[0];
|
||||
double y = arr[1];
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个经纬度点与区域的关系
|
||||
@@ -284,7 +301,7 @@ class InsidePolygon {
|
||||
if(flag){
|
||||
def arrs = key.split("##")
|
||||
name = arrs[1];
|
||||
break;
|
||||
// break;
|
||||
}
|
||||
}
|
||||
if(flag){
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package com.fangjia.script.majintao.test
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\1.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
count = count + 1
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
sleep(80)
|
||||
|
||||
def address = items['A']
|
||||
def map = SignatureUtils.getAliAddressStrc(address)
|
||||
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
|
||||
if ("prov" in map) {
|
||||
def province = map.get("prov")
|
||||
items['E'] = province
|
||||
}
|
||||
if ("city" in map) {
|
||||
def city = map.get("city")
|
||||
items['G'] = city
|
||||
}
|
||||
if ("district" in map) {
|
||||
def district = map.get("district")
|
||||
items['I'] = district
|
||||
}
|
||||
if ("town" in map) {
|
||||
def town = map.get("town")
|
||||
items['K'] = town
|
||||
}
|
||||
// if ("community" in map) {
|
||||
// def community = map.get("community")
|
||||
// items['S'] = community
|
||||
// }
|
||||
if ("devzone" in map) {
|
||||
def devzone = map.get("devzone")
|
||||
items['Q'] = devzone
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.majintao.JSBankTest;
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils;
|
||||
import net.sf.json.JSONObject;
|
||||
import java.util.Base64;
|
||||
import net.sf.json.JSONArray;
|
||||
|
||||
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
count += 1
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def zx = ["北京", "天津", "上海", "重庆"]
|
||||
// String province = "天津";
|
||||
String province = items['E'];
|
||||
if (zx.contains(province)) {
|
||||
province = province + "市";
|
||||
}
|
||||
// String name = "天津经济技术开发区";
|
||||
String name = items['Q'];
|
||||
String elasticUsername = "elastic";
|
||||
String elasticPassword = "elasticsearch";
|
||||
|
||||
String seawaterUrl = "http://192.168.0.89:9200/development_zone/_search";
|
||||
String basicAuth = "Basic " + Base64.getEncoder().encodeToString((elasticUsername + ":" + elasticPassword).getBytes("UTF-8"));
|
||||
|
||||
String body = """
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"name": {
|
||||
"query": "${name}",
|
||||
"operator": "and"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {"province": {"value": "${province}"}}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject jsonBody = JSONObject.fromObject(body);
|
||||
JSONObject header = new JSONObject();
|
||||
header.put("Authorization", basicAuth);
|
||||
String res = request.postWithBody(seawaterUrl, jsonBody, header, "UTF-8");
|
||||
|
||||
JSONObject response = JSONObject.fromObject(res);
|
||||
JSONObject obj = null;
|
||||
if (response != null && response.containsKey("hits")) {
|
||||
JSONObject hits = response.getJSONObject("hits");
|
||||
if (hits != null && hits.size() > 0) {
|
||||
JSONArray hitsArray = hits.getJSONArray("hits");
|
||||
// 省略了之前的部分...
|
||||
|
||||
if (hitsArray != null && hitsArray.size() > 0) {
|
||||
for (int i = 0; i < hitsArray.size(); i++) {
|
||||
JSONObject hit = hitsArray.getJSONObject(i);
|
||||
JSONObject source = hit.getJSONObject("_source");
|
||||
String devName = source.optString("name"); // Assuming "name" is a String
|
||||
String level = source.optString("level"); // Assuming "level" is a String
|
||||
if (devName.equals(name)) {
|
||||
items['O'] = "是";
|
||||
items['P'] = level;
|
||||
println("省份:" + province + " 名称:" + name + " 开发区名:" + devName + " 开发区级别:" + level);
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
});
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.majintao.JSBankTest
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.tools.ant.types.resources.selectors.None
|
||||
|
||||
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
count += 1
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
|
||||
|
||||
String address = items['B']
|
||||
|
||||
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
map.put("showFullValue", "2")
|
||||
map.put("showTestValue", "2")
|
||||
map.put("type", "2")
|
||||
|
||||
def content = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
def result = content.get("result")
|
||||
if (result != null) {
|
||||
if ("addrVerify") {
|
||||
items['C'] = result.get("addrVerify")
|
||||
}
|
||||
|
||||
if ("province" in result) {
|
||||
items['D'] = result.get("province")
|
||||
}
|
||||
if ("city" in result) {
|
||||
items['F'] = result.get("city")
|
||||
}
|
||||
if ("district" in result) {
|
||||
|
||||
items['H'] = result.get("district")
|
||||
}
|
||||
if ("town" in result) {
|
||||
items['J'] = result.get("town")
|
||||
}
|
||||
if ("committee" in result) {
|
||||
|
||||
items['L'] = result.get("committee")
|
||||
}
|
||||
if ("typeCode" in result) {
|
||||
items['M'] = result.get("typeCode")
|
||||
}
|
||||
if ("countyGov") {
|
||||
items['N'] = result.get("countyGov")
|
||||
}
|
||||
def devDistrict = ""
|
||||
if ("devDistrict" in result) {
|
||||
devDistrict = result.get("devDistrict")
|
||||
}
|
||||
def dev = ""
|
||||
if ("dev" in result) {
|
||||
dev = result.get("dev")
|
||||
}
|
||||
// 开发区级别暂时没有
|
||||
|
||||
// 暂时使用
|
||||
def defGrade = ""
|
||||
if (!dev.equals("") && devDistrict.equals("否")) {
|
||||
defGrade = "非国际级或省级"
|
||||
devDistrict = "是"
|
||||
}
|
||||
if (!devDistrict.equals("")) {
|
||||
items['O'] = devDistrict
|
||||
}
|
||||
items["P"] = defGrade
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
+35
@@ -5,7 +5,11 @@ import com.fangjia.script.gj.village.ailiyun.SignatureUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
|
||||
<<<<<<< HEAD
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx";
|
||||
=======
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
|
||||
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
@@ -14,13 +18,43 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
<<<<<<< HEAD
|
||||
sleep(80)
|
||||
=======
|
||||
sleep(50)
|
||||
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
|
||||
|
||||
def address = items['A']
|
||||
def map = SignatureUtils.getAliAddressStrc(address)
|
||||
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
|
||||
if ("prov" in map) {
|
||||
def province = map.get("prov")
|
||||
<<<<<<< HEAD
|
||||
items['C'] = province
|
||||
}
|
||||
if ("city" in map) {
|
||||
def city = map.get("city")
|
||||
items['G'] = city
|
||||
}
|
||||
if ("district" in map) {
|
||||
def district = map.get("district")
|
||||
items['K'] = district
|
||||
}
|
||||
if ("town" in map) {
|
||||
def town = map.get("town")
|
||||
items['O'] = town
|
||||
}
|
||||
if ("community" in map) {
|
||||
def community = map.get("community")
|
||||
items['S'] = community
|
||||
|
||||
}
|
||||
if ("devzone" in map) {
|
||||
def devzone = map.get("devzone")
|
||||
items['AE'] = devzone
|
||||
|
||||
}
|
||||
=======
|
||||
items['D'] = province
|
||||
}
|
||||
if ("city" in map) {
|
||||
@@ -66,6 +100,7 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
// items['K'] = roadno
|
||||
// }
|
||||
|
||||
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
|
||||
|
||||
})
|
||||
|
||||
|
||||
+114
@@ -4,10 +4,17 @@ import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
|
||||
<<<<<<< HEAD
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
=======
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
|
||||
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
|
||||
count += 1
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
@@ -15,6 +22,112 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
|
||||
String address = items['A']
|
||||
<<<<<<< HEAD
|
||||
// if (province == null) {
|
||||
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
map.put("showFullValue", "2")
|
||||
map.put("showTestValue", "2")
|
||||
map.put("type", "2")
|
||||
|
||||
def content = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
def result = content.get("result")
|
||||
if (result != null) {
|
||||
if ("province" in result) {
|
||||
items['B'] = result.get("province")
|
||||
}
|
||||
if ("provinceCode" in result) {
|
||||
items['E'] = result.get("provinceCode")
|
||||
}
|
||||
if ("city" in result) {
|
||||
items['F'] = result.get("city")
|
||||
}
|
||||
if ("cityCode" in result) {
|
||||
items['I'] = result.get("cityCode")
|
||||
}
|
||||
if ("district" in result) {
|
||||
|
||||
items['J'] = result.get("district")
|
||||
}
|
||||
if ("districtCode" in result) {
|
||||
|
||||
items['M'] = result.get("districtCode")
|
||||
}
|
||||
if ("town" in result) {
|
||||
|
||||
items['N'] = result.get("town")
|
||||
}
|
||||
if ("townCode" in result) {
|
||||
|
||||
items['Q'] = result.get("townCode")
|
||||
}
|
||||
|
||||
if ("committee" in result) {
|
||||
|
||||
items['R'] = result.get("committee")
|
||||
}
|
||||
if ("committeeCode" in result) {
|
||||
|
||||
items['U'] = result.get("committeeCode")
|
||||
}
|
||||
if ("typeCode" in result) {
|
||||
|
||||
items['V'] = result.get("typeCode")
|
||||
}
|
||||
if ("typeName") {
|
||||
items['W'] = result.get("typeName")
|
||||
}
|
||||
if ("verifyFlag") {
|
||||
items['X'] = result.get("verifyFlag")
|
||||
}
|
||||
if ("addrVerify") {
|
||||
items['Y'] = result.get("addrVerify")
|
||||
}
|
||||
if ("countyLevelCity") {
|
||||
items['Z'] = result.get("countyLevelCity")
|
||||
}
|
||||
if ("countyGov") {
|
||||
items['AA'] = result.get("countyGov")
|
||||
}
|
||||
if ("communitySource") {
|
||||
items['AB'] = result.get("communitySource")
|
||||
}
|
||||
|
||||
if ("administrativeTowns" in result) {
|
||||
items['AD'] = result.get("administrativeTowns")
|
||||
}
|
||||
if ("devDistrict" in result) {
|
||||
items['AF'] = result.get("devDistrict")
|
||||
}
|
||||
if ("devDistrictName" in result) {
|
||||
items['AG'] = result.get("devDistrictName")
|
||||
}
|
||||
if ("splitCommittee" in result) {
|
||||
items['AH'] = result.get("splitCommittee")
|
||||
}
|
||||
if ("splitStreet" in result) {
|
||||
items['AI'] = result.get("splitStreet")
|
||||
}
|
||||
if ("poiName" in result) {
|
||||
items['AJ'] = result.get("poiName")
|
||||
}
|
||||
if ("splitStreetNum" in result) {
|
||||
items['AK'] = result.get("splitStreetNum")
|
||||
}
|
||||
if ("dev" in result) {
|
||||
items['AL'] = result.get("dev")
|
||||
}
|
||||
}
|
||||
|
||||
// }
|
||||
|
||||
=======
|
||||
String province = items['B']
|
||||
if (province == null) {
|
||||
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
|
||||
@@ -120,5 +233,6 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
}
|
||||
|
||||
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
|
||||
})
|
||||
|
||||
|
||||
+5
@@ -24,8 +24,13 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
<<<<<<< HEAD
|
||||
map.put("complete", "2")
|
||||
map.put("addressAssessment", "2")
|
||||
=======
|
||||
map.put("complete", "true")
|
||||
map.put("addressAssessment", "true")
|
||||
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
|
||||
map.put("type", "2")
|
||||
|
||||
def content = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.fangjia.script.majintao.test
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
String url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
map.put("showFullValue", "2")
|
||||
map.put("showTestValue", "2")
|
||||
map.put("type", "2")
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.openApi
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import groovy.sql.Sql;
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
@@ -24,7 +25,7 @@ import com.fangjia.script.Watcher
|
||||
class OpenApiLogAnalyBatch extends BaseBatch{
|
||||
|
||||
|
||||
static DBCollection OPEN_URI_LOG_COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com",27020);
|
||||
static DBCollection OPEN_URI_LOG_COLL = com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI,"business", "open_uri_log")
|
||||
static DBCollection OPEN_URI_ANALY_COLL = MongoUtils.getCollectionByName("business", "open_uri_analy", "mongo.db.fangjia.com");
|
||||
final Sql USER_CONN = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?useUnicode=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver")
|
||||
|
||||
|
||||
+4
-4
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
|
||||
* 8:地址及地址别名查找
|
||||
* 9:地址及地址别名(不加区域限制)查找
|
||||
* 10:拼音匹配
|
||||
*11:open接口匹配
|
||||
* 11:open接口匹配
|
||||
* 14:拼音匹配,不加地址限制
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
@@ -61,10 +61,10 @@ class DataStandardBatch extends BaseBatch{
|
||||
private static def districtMap = [:];
|
||||
private static def limit = 5000;
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
//查找出所有未进行匹配的数据
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
|
||||
// qdata.put("city","福州")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
|
||||
+3
-3
@@ -23,8 +23,8 @@ class DataStat {
|
||||
// for(def city :citys){
|
||||
// println city
|
||||
// }
|
||||
// task.run();
|
||||
task.stat()
|
||||
task.run();
|
||||
// task.stat()
|
||||
|
||||
// def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"]
|
||||
// def rets = []
|
||||
@@ -76,7 +76,7 @@ class DataStat {
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def dates = [202301,202302,202303,202304,202305,202306,202307,202308,202309,202310,202311,202401]
|
||||
def dates = [202303,202304,202305,202306,202307,202308,202309,202310,202311,202401,202402]
|
||||
def sources = ["时间","我爱我家","兔博士","吉屋","贝壳找房","乐有家","365淘房","诸葛找房","链家","Q房","麦田","中原地产","安居客","搜房","C21","城市房产","幸福里"]
|
||||
// crawlCount(dates,sources);
|
||||
matchCount(dates,sources);
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||
// def esSources = ["搜房":"搜房"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDistrictPrice(), args);
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202307);
|
||||
def count2 = findTotalCount(source, 202401);
|
||||
def count1 = findTotalCount(source, 202401);
|
||||
def count2 = findTotalCount(source, 202402);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -32,7 +32,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
|
||||
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.KeyGenerator;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.util.Base64;
|
||||
|
||||
public class DESUtil {
|
||||
|
||||
private static final String ALGORITHM = "DES";
|
||||
|
||||
// 生成密钥
|
||||
public static SecretKey generateKey() throws Exception {
|
||||
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
|
||||
keyGenerator.init(56); // DES密钥长度为56位
|
||||
return keyGenerator.generateKey();
|
||||
}
|
||||
|
||||
// 将密钥转换为字符串
|
||||
public static String keyToString(SecretKey secretKey) {
|
||||
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
|
||||
}
|
||||
|
||||
// 将字符串转换为密钥
|
||||
public static SecretKey stringToKey(String keyStr) {
|
||||
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
|
||||
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
|
||||
}
|
||||
|
||||
// DES加密
|
||||
public static String encrypt(String data, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
|
||||
byte[] encryptedData = cipher.doFinal(data.getBytes());
|
||||
return Base64.getEncoder().encodeToString(encryptedData);
|
||||
}
|
||||
|
||||
// DES解密
|
||||
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey);
|
||||
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
|
||||
byte[] decryptedData = cipher.doFinal(decodedData);
|
||||
return new String(decryptedData);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
// 生成密钥
|
||||
// SecretKey secretKey = generateKey();
|
||||
// String keyStr = keyToString(secretKey);
|
||||
// System.out.println("密钥: " + keyStr);
|
||||
//
|
||||
//// // 加密数据
|
||||
// String data = "高娟";
|
||||
// String encryptedData = encrypt(data, secretKey);
|
||||
// System.out.println("加密后的数据: " + encryptedData);
|
||||
|
||||
// 解密数据
|
||||
SecretKey newSecretKey = stringToKey("tnb12345");
|
||||
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
|
||||
System.out.println("解密后的数据: " + decryptedData);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.*
|
||||
public class DbUtils{
|
||||
|
||||
public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com";
|
||||
public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
|
||||
//public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
|
||||
|
||||
public static final String COMMON_DB = "fangjia";
|
||||
public static final int COMMON_PORT = 27017;
|
||||
public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
|
||||
//public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
|
||||
public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
@@ -186,6 +186,26 @@ public class DbUtils{
|
||||
return query
|
||||
}
|
||||
|
||||
public static BasicDBObject buildSeaweedQuery2(def city = null, def region = null, def name = null, def cat = "district", def status = 0){
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
if(city){
|
||||
query.append("city", city)
|
||||
}
|
||||
if(region){
|
||||
query.append("region", region)
|
||||
}
|
||||
if(name){
|
||||
query.append("name", name)
|
||||
}
|
||||
if(cat != null){
|
||||
query.append("cat", cat)
|
||||
}
|
||||
if(status != null){
|
||||
query.append("status", status)
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
/**
|
||||
* °ÑÐ¡ÇøÐ޸ijÉɾ³ý״̬
|
||||
* @param id
|
||||
|
||||
+24
-1
@@ -76,7 +76,6 @@ class ElasticUtils {
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@@ -101,6 +100,30 @@ class ElasticUtils {
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 3
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 4
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
@@ -454,8 +454,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = 10
|
||||
println nThread
|
||||
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(3)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
final double total = body.size()
|
||||
|
||||
@@ -71,6 +71,10 @@ class MongoUtils {
|
||||
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
|
||||
}
|
||||
|
||||
if ((domainUrl == "192.168.0.206") && (port == 27020)) {
|
||||
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name)
|
||||
}
|
||||
|
||||
def key = db + name + domainUrl + port
|
||||
if (collectionMap.containsKey(key)) {
|
||||
return collectionMap[key]
|
||||
|
||||
@@ -156,6 +156,22 @@ public class ProvinceData {
|
||||
return PROVINCE_MAP.get(province);
|
||||
}
|
||||
|
||||
|
||||
public static String getSimpleProvince(String province){
|
||||
def prov = null
|
||||
def newDatas = data + GOVERNMENT_CITY_LIST
|
||||
for(def key : newDatas.keySet()){
|
||||
def list = newDatas.get(key)
|
||||
list << key
|
||||
if(list.contains(province)){
|
||||
prov = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
return prov;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取省份下的所有城市
|
||||
* @param province
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user