Merge remote-tracking branch 'origin/script_gj'

This commit is contained in:
majintao
2024-09-03 09:38:22 +08:00
1019 changed files with 49505 additions and 12759 deletions
@@ -3,13 +3,14 @@ package com.fangjia.script.ana
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
// 分期分区与主小区映射关系表
@BatchOwner(Watcher.WangWenYan)
class FindStageDistrictBatch extends BaseBatch {
@@ -24,7 +25,7 @@ class FindStageDistrictBatch extends BaseBatch {
@Override
void process(Object args) {
CommonUtils.findAllCity().each { def city ->
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
CommonDatautils.SEAWEED_COLL.find(DbUtils.buildSeaweedQuery2(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
if (dbo.name =~ STAGE_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
@@ -40,7 +41,6 @@ class FindStageDistrictBatch extends BaseBatch {
println(dbo)
}
}
}
}
}
@@ -21,6 +21,7 @@ import org.bson.types.ObjectId
/**
* 通知计算二手房均价
* 发送平米租金命令
* @author duxiaoyu
*
*/
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -72,21 +72,27 @@ for(def d : datas){
}
map.area = d['F'].toString().toDouble()
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
def trade_date = d['J']
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
map.area = d['F'].replace("平","").toString().toDouble()
map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble())
if(d['I']){
map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000)
}
map.trade_date = d['J']
// def trade_date = d['J']
// if(trade_date){
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
// }
// else{
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
// }
map.source = "深房家族"
map.create_date = new Date()
map.m_date = new Date()
println map
DEAL_PRICE_COLL.save(map)
}
@@ -23,7 +23,7 @@ class AddThirdPartyPrice {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def c : citys){
println c +" "+cityCodeDatas.get(c)
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c)
def rets = createDate(dists, distCodeMap)
println dists.size() +" "+rets.size()
@@ -0,0 +1,40 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
*/
class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new AddThirdPriceSample()
task.run()
}
def run(){
for(def i=1;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
if(d['A'] && d['C']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
println d['A'] +" 没查到"
}
}
}, i)
println(i)
}
}
}
@@ -24,15 +24,16 @@ class BatchAddPersonGuiPrice {
def run(){
def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx"
def path = "D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx"
def datas = ExcelUtils.read(path, 0, 1)
for(def d : datas){
def row = findGuidingPrice(d['B'], d['C'], d['D'])
def row = findGuidingPrice(d['B'], d['C'], d['C'])
if(!row){
def obj = CommonDatautils.findDistrictById(d['A'])
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
// def obj = CommonDatautils.findDistrictById(d['A'])
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
save(d['A'], d['B'], d['C'], d['Q'], obj.estate_type2, d['P'])
//发重算均价命令
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date())
addPrice(d['A'], d['B'], d['C'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),new Date())
}
}
}
@@ -43,13 +44,13 @@ class BatchAddPersonGuiPrice {
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("guiding_person" , "付贵生")
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd")))
// q.put("guiding_person" , person)
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-03-20", "yyyy-MM-dd")))
def dbo = GUIDING_PRICE_COLL.findOne(q)
return dbo
}
def save(city, region, name, price, estate_type){
def save(city, region, name, price, estate_type, person){
DBObject obj = new BasicDBObject();
obj.put("city", city);
obj.put("region", region);
@@ -62,7 +63,7 @@ class BatchAddPersonGuiPrice {
obj.put("m_date", new Date());
obj.put("estate_type" , estate_type);
obj.put("house_type" , 0)
obj.put("guiding_person" , "陈杰")
obj.put("guiding_person" , person)
GUIDING_PRICE_COLL.save(obj);
}
@@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null);
def distMap = [:]
for(def d : districts){
def alias = d.alias;
@@ -34,13 +34,15 @@ class BigdataCommunityBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def fangjiaDists = new HashSet()
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -52,14 +54,18 @@ class BigdataCommunityBatch extends BaseBatch{
def today = new Date().getDate()
println "starting 。。。。。。。。。"
def allDists = CcbCommonUtils.findAllDistrictByCity(null, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
for(def d : allDists){
fangjiaDists << d._id.toString()
}
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
println "加载建行小区量"
def ccbMaps = null
if(today<=10){
@@ -67,11 +73,11 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
}
println "组装新数据"
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()+" 结果:"+rows.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
@@ -103,10 +109,7 @@ class BigdataCommunityBatch extends BaseBatch{
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
@@ -34,7 +35,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
@@ -61,13 +62,21 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
// def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
// def objectId = dateToObjectId(date)
// println objectId
//664a22000000000000000000 5-20
//664b73800000000000000000 5-21
def citys =CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -180,7 +189,7 @@ class BigdataDealBatch extends BaseBatch{
})
}
println "查询完成......."
latch.await(5, TimeUnit.MINUTES);
latch.await();
return result
}
@@ -376,7 +385,7 @@ class BigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.region)
def data = distCodeMap.get(trade.match_region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
@@ -529,7 +538,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
@@ -679,6 +688,16 @@ class BigdataDealBatch extends BaseBatch{
return map
}
def updateCrawlTradeDate(){
def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20"
def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd")
def objectId = dateToObjectId(date)
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2)
def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1)
println(objectId)
CcbCommonUtils.updateTrades(startDate, endDate, objectId)
}
def addDealPriceMatch(city){
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
@@ -701,22 +720,33 @@ class BigdataDealBatch extends BaseBatch{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region) {
if(region || (!region && ["澄迈"].contains(t.city))) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
if(!rMap && t.address){
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
def flag = false
def address = null
if(t.address){
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
if(flag){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
address = t.address
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
if(!rMap && !t.name.toString().endsWith("小区")){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
}
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else {
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if (rMap && ["澄迈"].contains(rMap.city)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else{
if (rMap) {
println "区域不同"
}
@@ -733,7 +763,7 @@ class BigdataDealBatch extends BaseBatch{
}
}
);
)
}
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
@@ -816,4 +846,15 @@ class BigdataDealBatch extends BaseBatch{
println "end: "+new Date()
}
def dateToObjectId(Date date) {
// 获取时间戳(单位:秒)
def timestamp = (date.time / 1000) as int
def hexTimestamp = Integer.toHexString(timestamp)
// 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值
def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000'
return new ObjectId(objectIdStr)
}
}
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
//测试通过匹配接口配房源效果
@@ -7,17 +7,22 @@ import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.nio.charset.StandardCharsets
class CcbCommonUtils {
//
// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_fjw_all = ["长春","上海","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
@@ -69,6 +74,10 @@ class CcbCommonUtils {
citys = CcbCommonUtils.citys_252
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="all"){
citys = CcbCommonUtils.citys_fjw_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
}else{
citys = CcbCommonUtils.citys_all
@@ -228,7 +237,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -428,16 +437,19 @@ class CcbCommonUtils {
return rows
}
public static def findAllDistrictByCity(def city, def cat, def fields){
public static def findAllDistrictByCity(def city, def cat, def fields, def kind){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
if(city){
q.put("city", city);
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","汤堡小区")
// q.put("name","一米阳光一期")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
DBObject keys = new BasicDBObject();
if(fields){
@@ -445,14 +457,13 @@ class CcbCommonUtils {
keys.put(field, 1);
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
def rets = new HashSet()
def ids = new HashSet()
def ids = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(kind){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
@@ -460,6 +471,16 @@ class CcbCommonUtils {
ids << d._id.toString()
rets << d
}
}else{
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
if(!ids.contains(d._id.toString())){
ids << d._id.toString()
rets << d
}
}
}
}
println "[${city}] 小区量:"+rets.size()
@@ -620,7 +641,8 @@ class CcbCommonUtils {
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("match_id", new BasicDBObject('$exists', false))
// q.put("district_name","中海寰宇天下")
// q.put("district_name","新明胡同")
// q.put("address","桂苑大道135号")
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
@@ -665,15 +687,22 @@ class CcbCommonUtils {
}
public static def deleteGuidedPrice(def dealdate){
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'"
pgsqlInstance.executeUpdate(sql2)
}
public static def findGuiDingIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
pgsqlInstance.eachRow("SELECT gujia_fsid FROM base.t0831_guided_price_gj_h",{it->
filter.put(it.gujia_fsid)
});
return filter
}
public static def findDistGovPrice(def city){
def ids = []
pgsqlInstance.eachRow("SELECT gujia_bds_id from base.t0831_community_guide_price_gj_h where urbn_nm='${city}'",{it->
@@ -702,4 +731,18 @@ class CcbCommonUtils {
return map
}
public static def updateTrades(startDate, endDate, cDate){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("_id", new BasicDBObject('$gte', cDate))
def count = DEAL_PRICE_COLL.count(q)
if(count>0){
def udata = new BasicDBObject()
udata.put("trade_date", endDate)
udata.put("m_date", new Date())
udata.put("flag", "补漏变更成交时间")
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
}
}
}
@@ -387,26 +387,45 @@ class DealPriceDataProcess {
}
def deleteThirdPartyData(){
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
def citys = CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
def total = 0
def cityid = cityCodeDatas.get(city)
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
println city +" "+rets.size()
if(rets){
set.add(city)
}
for(def m : rets){
def cityId = cityCodeDatas.get(city)
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
def total = 0
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityId}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
}
}
}
println set.toString()
@@ -432,8 +451,8 @@ class DealPriceDataProcess {
return rets
}
def findPeGteThirdParty2(cityName, yearmonth){
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>20"
def findPeGteThirdParty2(cityid, yearmonth){
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
@@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -47,16 +47,22 @@ class GuiDingPriceBatch extends BaseBatch{
def rows = []
CcbCommonUtils.deleteGuidedPrice(curDate)
def priceIdsfilter = CcbCommonUtils.findGuiDingIdsByCity()
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
// def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
//近3个月人工指导价
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
rows.addAll(rets)
println city + " " + rets.size()
}
@@ -80,7 +86,8 @@ class GuiDingPriceBatch extends BaseBatch{
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
println "人工指导价总计:"+rows.size()
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
saveToPgSql(rows)
//
@@ -142,11 +149,18 @@ class GuiDingPriceBatch extends BaseBatch{
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(priceIdsfilter.mightContain(ps._id.toString())){
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
continue
}
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
@@ -207,7 +221,7 @@ class GuiDingPriceBatch extends BaseBatch{
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -2,6 +2,9 @@ package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
/**
* ½¨ÐÐʹÓÃ
*/
class PgsqlCommUtils {
public static def database = "warehouse"
@@ -19,6 +22,9 @@ class PgsqlCommUtils {
}
public static def findAllDistrictCount(def cityid){
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def count = 0
@@ -27,4 +33,13 @@ class PgsqlCommUtils {
});
return count
}
public static def findDistrictMap(){
def sql = "select urbn_id,gujia_admndiv_id,gujia_admndiv_nm from base.t0831_bigdata_district_gj_h"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.put(r.urbn_id+"#"+r.gujia_admndiv_id, r.gujia_admndiv_nm)
});
return map
}
}
@@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null)
def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap)
rows.addAll(rets)
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
@@ -198,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
}
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
// if(d['G']){
// def obj = CommonDatautils.findDistrictById(d['G'])
// if(obj){
// d['H'] = obj.name
// }
// }
//})
////def url = "https://sh.anjuke.com"
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
@@ -601,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
////},0)
//
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
// if(d['A']){
// def dbo = CommonDatautils.findDistrictById(d['D'])
// def dbo = CommonDatautils.findDistrictById(d['A'])
// if(dbo){
// def udata = new BasicDBObject()
//// udata.put("name_cat", 11)
@@ -619,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
// }
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){
def alias = null
if(!dbo.alias){
alias = []
}else{
alias = dbo.alias
}
if(!alias.contains(d['B'])){
alias << d['B']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d['A']+" "+"别名已存在"
}
}else{
println dbo._id.toString() +" 没找到"
}
}
})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
@@ -1288,35 +1331,158 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
// if(dist){
// d['G'] = dist.region
// d['H'] = dist._id.toString()
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
//
//// d['H'] = dist.name
// }
//// }
// }
//},0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
//ExcelUtils.multiThreadingModify(file, { d ->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['C'] = dist.city
// d['D'] = dist.region
// d['E'] = dist.name
// def random = RandomUtil.random(-300,50)
// d['F'] = random + d['B']
// }
// }
//})
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
//ExcelUtils.multiThreadingModify(filePath, { d ->
// if(d['B']){
//
// def obj = CommonDatautils.findDistrictById(d['B'])
// if(obj){
// def markMap = obj.mark_map
// if(!obj.mark_map){
// markMap = [:]
// }
// def data = new BasicDBObject()
// markMap.put("exclusive", "是")
// data.put("mark_map", markMap)
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
// }
// }
//
//})
//
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
//ExcelUtils.modify(filePath, { d ->
// if(d['A']){
//// def obj = CommonDatautils.findDistrictById(d['A'])
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
// if(list){
// if(list.size()==1){
// def dbo = list[0]
// if(dbo){
// def data = new BasicDBObject()
// data.put("region", d['B'])
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
// }
// }else{
// println d['C'] +" 有多个"
// }
// }
if(!d['I']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
d['F'] = dist.region
// d['H'] = dist.name
d['G'] = dist.status
// if(d['A']=="天门" || d['A']=="迪庆"){
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
// d['K'] = val.toInteger()
// }
}
}
}
},0)
//// if(obj){
//// def lng = d['G']
//// def lat =d['H']
////
//// // 高德转百度
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
//// def lng2 = baidu.x
//// def lat2 = baidu.y
////
////
//// def data = new BasicDBObject()
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
//// data.put("lng", lng_new)
//// data.put("lat", lat_new)
////
//// Long dLat2 = lat2 * 10000000000L
//// Long dLng2 = lng2 * 10000000000L
//// data.put("lat2", dLat2)
//// data.put("lng2", dLng2)
////
////// data.put("status", 1)
//// data.put("m_date", new Date())
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
//// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// if(d['B'] && !d['D']){
// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
// if(jsonObject != null && jsonObject.credit>0) {
// d['D'] = jsonObject.get("city")
// d['E'] = jsonObject.get("district")
// d['F'] = jsonObject.get("name")
// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
// if(dbo){
// d['G'] = dbo.status
// d['H'] = dbo.name_cat
// d['I'] = dbo.estate_type2
// d['J'] = "接口匹配"
// }
// }
//
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
//// if(!objs){
//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
//// }
//// if(objs && objs.size()==1){
//// def o = objs[0]
//// if(o){
//// d['D'] = o.city
//// d['E'] = o.region
//// d['F'] = o.name
//// d['G'] = o.status
//// d['H'] = o.name_cat
//// d['I'] = o.estate_type2
//// d['J'] = "别名"
//// }
//// }
//
//
// }
//})
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
//// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
@@ -1379,6 +1545,9 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
return count
}
//citys = CcbCommonUtils.citys_all
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
@@ -0,0 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
if(d['G'] && d['G'] instanceof String){
def obj = CommonDatautils.findDistrictById(d['G'])
if(obj){
d['H'] = obj.name
}
}
})
@@ -47,14 +47,6 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
// def gocPrices = CcbCommonUtils.findDistGovPrice("深圳")
// def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val=202303 and gujia_bds_id=?"
// pgsqlInstance.withBatch(500, sql){ ps->
// for(def id : gocPrices){
// ps.addBatch(id)
// }
// }
//
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
@@ -71,7 +63,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def statMap = getDistrictStatByCity(city, weekend)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
def thirdPriceMap = findThirdPartyPrice(city, statMap)
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
@@ -265,6 +257,12 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
if(third.get("csjj_esf_price")){
price_map.csjj_avg_price = third.get("csjj_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -311,6 +309,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "房产超市"
}else if(k=="csjj_avg_price"){
web = "城市居家"
}
ret << sourceMap.get(web)
@@ -528,6 +530,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
@@ -592,6 +595,14 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -599,8 +610,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
if(value && value.size()>1){ //csjj_esf_price 国信达成交
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price && !value.csjj_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
@@ -32,6 +32,10 @@ class TradeStatLog {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -40,8 +44,10 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -499,8 +505,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2023-08-01"
def endDate = "2023-10-01"
def startDate = "2024-04-01"
def endDate = "2024-06-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -584,6 +590,36 @@ class TradeStatLog {
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def exportTrade2(){
// def startDate = "2024-06-01"
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
if(d['S'] && !d['W']){
def dist = CommonDatautils.findDistrictById(d['S'])
if(!dist.name_cat && dist.name_cat!=0){
d['W'] = "name_cat过滤"
}
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
d['W'] = "小区过滤"
}
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
def map = findDealByCommId(d['B'], d['S'])
def count = map.get(key)
if(count>=2){
d['W'] = "重复"
}
}
println d['A']
})
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
@@ -637,11 +673,39 @@ class TradeStatLog {
return trades
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
ids << r.gujia_bds_mdl_id
}
})
return ids
}
def findDealByCommId(city, commid){
def q = new BasicDBObject()
q.put("match_id", commid)
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
def dbcursor = DEAL_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
return map
}
def findDealIds(city, startDate, endDate){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
def size = regionDatas.size()
final CountDownLatch latch = new CountDownLatch(size)
def totalCount = new AtomicLong()
for(def i = 0;i<regionDatas.size();i++){
final def arr = regionDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
totalCount.addAndGet(1)
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
if(row){
rets << row
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// };
// })
}
latch.await()
// latch.await()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
def allDistRegionPreMap = [:]
def allLastMonthMap = [:]
def allLastYearMap = [:]
def allCcbPriceMap = [:]
def allCcbPriceCaseCountMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
allDistricts.addAll(districts)
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
allPriceMap = allPriceMap + priceMap
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
// allPriceMap = allPriceMap + priceMap
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
allRentMap = allRentMap + rentMap
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
allDealCountMap = allDealCountMap + dealCountMap
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceMap = allCcbPriceMap + datasMap
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
def cityRankMap = findCommunityCityRank(datas)
allCityRankMap = allCityRankMap + cityRankMap
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
allLastYearMap = allLastYearMap + lastYearMonthMap
}
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(cityLists){
rets.addAll(cityLists)
}
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def totalCount = new AtomicLong()
def distCount = districts.size()
final CountDownLatch latch = new CountDownLatch(distCount)
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
void run() {
try {
totalCount.addAndGet(1)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(row){
rets << row
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def sellDataType = 1
def rentDataType = 1
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
//房源查询是否按建行均价筛选房源
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
def sell_avg_price = null
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def sell_count = 0
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
if(allCcbPriceMap.containsKey(dist._id.toString())){
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
}
}
if(sell_count){
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
}
}
}
if(sellMap){
sell_count = sellMap.avg_price.size()
if(sell_count>3){
sell_avg_price = avgValue(sellMap.avg_price)
if(sellMap){
sell_avg_area = avgValue(sellMap.area)
sell_avg_total = avgValue(sellMap.total_price)
}else{
sell_count = 0
sellDataType = null
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// if(sell_count>3){
// if(allCcbPriceMap.containsKey(dist._id.toString())){
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
// }else{
// sell_count = 0
// sellDataType = null
// }
// }else{
// sell_count = 0
// sellDataType = null
// }
}
}
def rent_avg_price = null
def rent_avg_area = null
def rent_avg_total = null
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
rentDataType = null
}
}
if(!sell_avg_price){
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
// if(!sell_avg_price){
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
// }
if(!rent_avg_price){
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def avgPrice = null
if(region){
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
}else{
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
}
if (sell_count < 10) {
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
sellStartDate = startDate
}
if (sell_count) {
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
}
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
return datas;
}
def findCommunityPriceByCityMap(cityCode, month){
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
})
return dataMap;
}
def findCommunityPriceCountByCityMap(cityCode, month){
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
if(r.gujiabdsrt3mlgcasenum){
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
}
})
return dataMap;
}
def findCcbCityPrice(cityCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def avgPrice = null
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
return regionPriceMap;
}
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
def price = null
pgsqlInstance.eachRow(sql, { r ->
price = r.gujia_bds_clc_avprc
})
return price;
}
def districtCityPrictRate(cityPrice, datas){
def distCityRateMap = [:]
@@ -159,7 +159,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
}
@@ -246,7 +246,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
}
@@ -304,7 +304,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
}
@@ -362,7 +362,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
}
@@ -425,7 +425,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
}
@@ -504,7 +504,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
}
@@ -590,7 +590,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
rets << ret
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
}
@@ -4,17 +4,15 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import java.sql.ResultSet
import java.sql.ResultSetMetaData
/**
* 建行融合数据导出
*/
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHe extends BaseBatch{
@@ -69,6 +67,12 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
// 新增两张数据表
// avm_evaluate_rate_city_gj_h
// avm_evaluate_rate_comm_gj_h
exportEvaluateRateCity(queryDate)
exportEvaluateRateComm(queryDate)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
// exportCommunityBasePrice(lastMonth)
// exportCommunityPdescribeGj(lastMonth)
@@ -142,7 +146,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
@@ -217,7 +221,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
@@ -228,6 +232,80 @@ class ExportDistrictRongHe extends BaseBatch{
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
// avm_evaluate_rate_city_gj_h gujia_clc_yrmo_val =202402 and gujia_clc_yrmo_val<=202403 gujia_clc_yrmo_val>=to_date('${startDateStr}','yyyy-MM-dd') and gujia_clc_yrmo_val<=to_date('${endDateStr}','yyyy-MM-dd')
def exportEvaluateRateCity(queryDate){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_bld_tp_val
data << r.gujia_bld_tp_val_coef
data << r.gujia_clc_yrmo_val
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 上传Ftp完成......"
}
def exportEvaluateRateComm(queryDate){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' ";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujia_clc_yrmo_val
data << r.basic_price
data << r.area_avg
data << r.unitprc_avg
data << r.comm_type
data << r.area_avg_coef
data << r.unitprc_avg_coef
data << r.basic_price_coef
data << r.buildarea_coef
data << r.type_coef
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
@@ -249,7 +327,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
@@ -278,7 +356,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
@@ -306,7 +384,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
@@ -344,7 +422,7 @@ class ExportDistrictRongHe extends BaseBatch{
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
@@ -374,7 +452,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
@@ -407,7 +485,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
@@ -439,7 +517,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
@@ -474,7 +552,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
@@ -498,7 +576,7 @@ class ExportDistrictRongHe extends BaseBatch{
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
@@ -598,7 +676,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -638,7 +716,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -909,7 +987,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -920,7 +998,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
@@ -994,7 +1072,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
@@ -1006,7 +1084,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportThirdPartyPrice2(price_date, new_dists) {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1054,7 +1132,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgThirdPartyPrice2(price_date, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1137,7 +1215,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
@@ -1183,7 +1261,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
@@ -1226,7 +1304,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1241,7 +1319,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1315,7 +1393,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1382,7 +1460,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1423,7 +1501,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1459,7 +1537,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1492,7 +1570,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1531,7 +1609,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1574,7 +1652,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1611,7 +1689,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1659,7 +1737,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1706,7 +1784,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1767,7 +1845,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
@@ -1778,7 +1856,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataForsale(publishDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1835,7 +1913,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataForsale(publishDate,endDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -2020,7 +2098,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
def dataCodeFileName = "/home/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
@@ -1,6 +1,7 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
@@ -8,11 +9,18 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.poi.ss.util.RegionUtil
import org.bson.Document
import org.bson.types.ObjectId
import com.fangjia.core.util.HttpRequest;
@@ -40,6 +48,8 @@ import java.util.zip.GZIPInputStream
class CommonDatautils {
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -108,6 +118,9 @@ class CommonDatautils {
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -265,7 +278,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","泰瑞家园")
// q.put("name","托斯卡纳庄园")
DBObject keys = new BasicDBObject();
if(fields){
@@ -274,7 +287,7 @@ class CommonDatautils {
}
}
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -1175,6 +1188,35 @@ class CommonDatautils {
return null;
}
public static def findAllStatusDistrict(String city,String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("region", region);
q.put("name", name);
q.put("cat", cat);
def rets = []
def cursor = SEAWEED_MONGO_COLL.find(q);
if(cursor){
for(def obj : cursor){
rets << obj;
}
}
return rets;
}
public static def findAllCount(String cat){
DBObject q = new BasicDBObject();
q.put("cat", cat);
q.put("status", 0)
return SEAWEED_MONGO_COLL.count(q)
}
public static def findDistrictsByName(String city, String name, String cat){
DBObject q = new BasicDBObject();
@@ -1435,6 +1477,28 @@ class CommonDatautils {
return obj;
}
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('status',0)
BasicDBObject query = new BasicDBObject();
query.putAt(QueryOperators.IN, [name]);
q.put("mapping_alias", query);
def dbcursor = SEAWEED_COLL.find(q);
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
static def findSeaweeds(city, cat, name_cat){
DBObject q = new BasicDBObject();
q.put('city',city)
@@ -1537,6 +1601,32 @@ class CommonDatautils {
}
static def findAllStatusDistrictByAlias(def city, def region, def name, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
def names = [];
names << name;
BasicDBObject query = new BasicDBObject();
query.putAt(QueryOperators.IN, names);
q.put("alias", query);
def dsCursor = getDataStoreConnection().find(q);
def ret = [];
for(def obj : dsCursor){
ret << obj;
}
return ret;
}
static def findDistrictByAddress(def city, def region, def address, def cat){
DBObject q = new BasicDBObject();
if(region){
@@ -1641,7 +1731,34 @@ class CommonDatautils {
}
return regionMap;
}
public static def getAllCityFullRegions(){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("category","region");
//queryCondition.put("show",1);
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
def regionMap = [:];
for(DBObject obj : dbCursor){
def city = obj.get("city");
def region = obj.get("name");
def pattern = obj.get("pattern");
if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) {
def r_map = regionMap.get(city);
if(!r_map){
r_map = [:]
regionMap.put(city, r_map);
}
if(!pattern){
pattern = [];
pattern << region;
}
r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]);
}
}
return regionMap;
}
public static def getRegionsByCity(def city){
@@ -1737,39 +1854,6 @@ class CommonDatautils {
}
return priceMap;
}
public static def queryDistrictPrice(city, region, name, weekend, cat){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
if(cat == "district"){
BasicDBObject queryObj = new BasicDBObject();
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
queryCondition.put("estate_type", queryObj);
}else if(cat == "office"){
queryCondition.put("estate_type", "写字楼");
}else if(cat == "shop"){
queryCondition.put("estate_type", "商铺");
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public static def queryFirstDistrictPrice(city, region, name){
@@ -1784,11 +1868,63 @@ class CommonDatautils {
def dbo = null;
while(cur.hasNext()) {
dbo = cur.next()
if(dbo.value_type == 1){
dbo.cal_type = "指导价"
}else if(dbo.value_type == 2){
dbo.cal_type = "自身样本"
}else if(dbo.value_type == 3){
dbo.cal_type = "第三方"
}else if(dbo.value_type == 4){
dbo.cal_type = "交易数据"
}else if(dbo.value_type == 5){
dbo.cal_type = "周边楼盘"
}else if(dbo.value_type == 6){
dbo.cal_type = "推导"
}else if(dbo.value_type == 9){
dbo.cal_type = "神经网络"
}else if(dbo.value_type == 10){
dbo.cal_type = "无方式"
}else{
dbo.cal_type = ""
}
}
return dbo;
}
public static def queryDistrictPrice(city, region, name, weekend, cat){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
if(cat == "district"){
BasicDBObject queryObj = new BasicDBObject();
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
queryCondition.put("estate_type", queryObj);
}else if(cat == "office"){
queryCondition.put("estate_type", "写字楼");
}else if(cat == "shop"){
queryCondition.put("estate_type", "商铺");
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public static def queryDistrictPrice(city, region, name){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
@@ -3764,6 +3900,135 @@ class CommonDatautils {
return rets;
}
static def findHouses2(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(true);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range).must(area_range)
if(region){
queryBuilder.must(QueryBuilders.termQuery("region", region))
}
if(name || alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def avgPrices = []
def areas = []
def totalPrices = []
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(cat == "rent"){
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
avgPrices << seawater.avg_price
areas << seawater.area
totalPrices << seawater.total_price
}
}
def retMap = [:]
if(areas){
retMap.avg_price = avgPrices
retMap.area = areas
retMap.total_price = totalPrices
}
return retMap;
}
/**
* 是否是别墅
* @return
*/
public static boolean isVilla(Double area, String name, Integer height, Integer room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
}else if(height != null){
if (height <= 3) {
if (area > 200){
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
@@ -3919,6 +4184,29 @@ class CommonDatautils {
return avg_price
}
static def findMatchTrades(city, region, name, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
/**
@@ -3993,6 +4281,32 @@ class CommonDatautils {
return total;
}
static def findLogs(user_id, requestUri, startDate, endDate){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", user_id)).must(QueryBuilders.termQuery("request_uri", requestUri))
if(startDate){
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
queryBuilder.must(date_range_cdate);
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
if(map.parameter_map){
def pMap = map.parameter_map;
if((pMap.keyword && pMap.keyword.contains("紫金山蘭园")) || (pMap.name && pMap.name.contains("紫金山蘭园"))){
println map
}
}
}
}
return null;
}
static def findNewAmapDetailBySql(city){
String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
@@ -4219,6 +4533,23 @@ class CommonDatautils {
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("xingfuli_esf_price") && db.xingfuli_esf_price && db.xingfuli_esf_price>3000){
dataMap.put("xingfuli_esf_price", db.xingfuli_esf_price)
}
if(!dataMap.containsKey("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
}
return map
}
@@ -4251,6 +4582,28 @@ class CommonDatautils {
return townMap
}
static def findAllTownRanges(){
def q = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("town", 1)
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
def townMap = [:]
for(def d : dbcursor){
def lists = townMap.get(d.city)
if(!lists){
lists = []
}
def t = [:]
t.region = d.region
t.town = d.town
lists << t
townMap.put(d.city, lists)
}
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
@@ -4285,6 +4638,148 @@ class CommonDatautils {
}
static def queryFullNameByCity(){
def q = new BasicDBObject()
q.put("type","城市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def cityMap = [:]
for(def d : dbcursor){
def city = RegionData.formatCity(d.city)
if(city){
cityMap.put(city, d)
}else{
//println d.city +" 格式化不出"
}
}
return cityMap
}
static def queryDistNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
keys.put("district_short",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def regionMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
def regions = regionMap.get(d.city)
if(!regions){
regions = []
}
regions << d
regionMap.put(d.city, regions)
}else{
println d.province+" "+d.district +" 格式化不出民政部区域"
}
}
println "区域加载完成"
return regionMap
}
static def queryCommunityByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "乡镇街道 加载完成"
return townMap
}
static def queryTownByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
def key = d.city
def list = townMap.get(key)
if(!list){
list = []
}
def t = [:]
t.region = d.district
t.town = d.street
list << t
townMap.put(key, list)
}
return townMap
}
static def queryDistNameCity(){
def q = new BasicDBObject()
@@ -4312,6 +4807,21 @@ class CommonDatautils {
}
static def queryDistNameCityByMapping(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.district_short && d.city_short){
retMap.put(d.city_short+"#"+d.district_short, d)
}
}
return retMap;
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
@@ -4337,6 +4847,111 @@ class CommonDatautils {
}
static def findHouseAvgPrice(city, region , start, end){
def prices = []
def sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date > ${start} and p_date<${end} and (root='lianjia.com' or root='ke.com')"
if(region){
sql = sql + " and region='${region}'"
}
ElasticUtils.executeSqlQuery(sql, {
prices << it.avg_price
})
if(prices && prices.size()>3){
def avg_price = (int)(prices.sum()/prices.size())
println region + " "+ prices.size() +" "+avg_price +" "+start+" "+end
return avg_price
}
return null
}
static def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
def road = ""
def name = ""
def type = ""
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
if(addressComponent.streetNumber){
if(addressComponent.streetNumber.street){
road = addressComponent.streetNumber.street
}
if(addressComponent.streetNumber.number){
road = road + addressComponent.streetNumber.number
}
}
if(addressComponent.building){
if(addressComponent.building.name){
name = addressComponent.building.name
}
if(addressComponent.building.type){
type = addressComponent.building.type
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
map.road = road
map.type = type
map.name = name
return map;
}
static def saveOptLog(city, region, name, cat, opt, user_name){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category",cat);
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", opt)
obj.put("user_name", user_name)
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -44,12 +44,17 @@ class DistrictHouseBatch extends BaseBatch {
public void process(Object args) {
// DBObject q = new BasicDBObject();
// q.put("_id", new ObjectId("6501174f6f5638447b264189"));
// def obj = CommonDatautils.getHouseDataStoreConnection().findOne(q)
// obj.put("c_date", new Date())
// CommonDatautils.getHouseDataStoreConnection().save(obj)
// q.put("cat", "sell");
// q.put("city","常州")
// q.put("district_name","清河苑一区")
// def dbcursor = CommonDatautils.getHouseDataStoreConnection().find(q)
// for(def obj : dbcursor){
// obj.put("c_date", new Date())
// CommonDatautils.getHouseDataStoreConnection().save(obj)
// }
// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","新城帝景南区_武进_常州")).each{
// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","清河苑一区_溧阳_常州")).each{
// println(it)
// }
@@ -62,14 +67,15 @@ class DistrictHouseBatch extends BaseBatch {
c++
try {
long sn = Long.parseLong("${new Date().format('yyyyMMddHHmmssSSS')}")
List<DBObject> districts = findDistrictByCity(city)
// List<DBObject> districts = findDistrictByCity(city)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"])
Map<String, Long> sellCountMap = facetDistrictInHouse("sell", city);
Map<String, Long> rentCountMap = facetDistrictInHouse("rent", city);
println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
for(DBObject district : districts) {
def names = []
names << district.name
if(district.alias && districts.size()>0){
if(district.alias && districts.alias.size()>0){
for(def name : district.alias){
if(!names.contains(name)){
names << name
@@ -88,11 +94,11 @@ class DistrictHouseBatch extends BaseBatch {
rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0
}
}
saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn)
}
HOUSE_COLL.remove(new BasicDBObject("city", city).append('sn', new BasicDBObject('$ne', sn)))
println c + " " + city
}catch (Exception e) {
e.printStackTrace();
}
@@ -151,7 +157,7 @@ class DistrictHouseBatch extends BaseBatch {
q.put('city', city)
q.put('cat', 'district')
q.put('visible', 0)
// q.put("name","新城帝景")
q.put("name","清河苑")
DSCursor cur = SEAWEED.find(q, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
@@ -0,0 +1,3 @@
package com.fangjia.script.fuzhou
@@ -7,6 +7,7 @@ import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -74,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
def day = new Date().getDate()
def startMDate = DateUtils.truncateDay(new Date()-1)
def startMDate = DateUtils.truncateDay(new Date()-7)
def endMDate = DateUtils.truncateDay(new Date())
def index = 0
@@ -144,7 +145,8 @@ class DistrictSellStatBatch extends BaseBatch {
void run() {
try {
totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name
def type = dist.estate_type2=="别墅"?"别墅":"小区"
def key = dist.region +"##" + dist.name+"##"+ type
def weekendPriceObj = weekendPriceMap.get(key)
if(weekendPriceObj){
@@ -153,8 +155,13 @@ class DistrictSellStatBatch extends BaseBatch {
def lastYearPriceObj = lastYearPriceMap.get(key)
double mom = 0d
if(lastMonthPriceObj) {
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
if(lastMonthPriceObj && lastMonthPriceObj.avg_price>0) {
try{
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
}catch(Exception e){
println key+" 出错................."
}
}
def curPrice = weekendPriceObj.avg_price
@@ -192,7 +199,7 @@ class DistrictSellStatBatch extends BaseBatch {
curPrice = 0
}
if(curPriceObj){
if(curPrice!=weekendPriceObj.avg_price){
if(curPrice!=curPriceObj.avg_price){
def data = new BasicDBObject()
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
@@ -257,13 +264,13 @@ class DistrictSellStatBatch extends BaseBatch {
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
def priceMap = [:]
for(def d : dbcursor){
if(city == "¸£ÖÝ"){
if(d.avg_price_2b && d.value_type!=1){
d.avg_price = d.avg_price_2b
if(!priceMap.containsKey(d.region +"##"+d.district_name+"##"+d.estate_type)){
if(city == "福州"){
if(d.avg_price_2b && d.value_type!=1){
d.avg_price = d.avg_price_2b
}
}
}
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
priceMap.put(d.region +"##"+d.district_name, d)
priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d)
}
}
return priceMap
@@ -276,7 +283,12 @@ class DistrictSellStatBatch extends BaseBatch {
def dbcursor = DISTRICT_SELL_STAT.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.region +"##"+d.district_name, d)
if(d.estate_type=="别墅"){
priceMap.put(d.region +"##"+d.district_name+"##别墅", d)
}else{
priceMap.put(d.region +"##"+d.district_name+"##小区", d)
}
}
return priceMap
}
@@ -355,4 +367,5 @@ class DistrictSellStatBatch extends BaseBatch {
// }
}
@@ -9,6 +9,7 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
@@ -29,9 +30,10 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
task.regionPriceTrendRun()
// task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
// task.testNewPrice()
}
def margeExcel(){
@@ -56,7 +58,7 @@ class FzDataResultStat {
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
@@ -80,7 +82,7 @@ class FzDataResultStat {
}
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth)
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
def map2 = [:]
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
@@ -163,7 +165,7 @@ class FzDataResultStat {
def exportRegion(city, months){
def regions = ["뮬짜","꾑�","憩쉭","쐈갛","쯩棺","낀있"];
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"];
def datas =[]
def header = ["城市", "区域"]
for(def m : months){
@@ -206,8 +208,12 @@ class FzDataResultStat {
DBObject dbo = cur.next()
prices << dbo.get("avg_price");
}
def avg_price = (int)(prices.sum()/prices.size())
return avg_price;
if(prices){
def avg_price = (int)(prices.sum()/prices.size())
return avg_price;
}
return null;
}
@@ -293,4 +299,20 @@ class FzDataResultStat {
}
return rets
}
def testNewPrice(){
ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d->
if(d['A']){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region",d['B'])
q.put("name",d['C'])
q.put("month", 202402)
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
d['S'] = obj.avg_price
}
}
})
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -19,7 +20,72 @@ class FzDistrictMatch {
def task = new FzDistrictMatch()
// task.run();
// task.addPrice()
task.addThirdPrice()
// task.addThirdPrice()
task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
// task.longyanPrice()
}
def longyanPrice(){
// def dists = CommonDatautils.findAllDistricts("龙岩","district", ["city","region","name","estate_type2"])
// def rets = []
// rets << ["Id", "城市","区域", "名称", "物业类型", "上个月价格toB","上个月价格","最新价格toB","最新价格"]
// for(def d : dists){
// def ret = []
// ret << d._id.toString()
// ret << d.city
// ret << d.region
// ret << d.name
// ret << d.estate_type2
// def obj2 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-03-10"), 1)
// if(obj2){
// ret << obj2.avg_price_2b
// ret << obj2.avg_price
// }
//
// def obj1 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-04-07"), 1)
// if(obj1){
// ret << obj1.avg_price_2b
// ret << obj1.avg_price
// }
// rets << ret
// }
// ExcelUtils.write("D:\\数据\\福州\\龙岩价格对比.xlsx", rets, "龙岩")
addThirdPrice("D:\\数据\\福州\\龙岩价格对比.xlsx")
}
def addThirdPrice2(path){
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice(row['A'], row['B'], row['C'], 202312, 202402)
def priceMap = cityMap.get(row['A']+"##"+row['B']+"##"+row['C'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['J'] = priceMap.get("lianjia_esf_price")
}
if(!row['J'] && priceMap.containsKey("beike_esf_price")){
row['J'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("maitian_esf_price")){
row['K'] = priceMap.get("maitian_esf_price")
}
}
def tradeAvgPrice = CommonDatautils.findMatchTrades(row['A'], row['B'], row['C'], "2023-08-01", "2024-01-31");
if(tradeAvgPrice){
row['L'] = (int)tradeAvgPrice
}
def priceObj = CommonDatautils.findDistrictPrice(row['A'], row['B'], row['C'])
if(priceObj){
row['M'] = priceObj.avg_price_2b
}
})
}
def addThirdPrice(){
@@ -173,4 +239,57 @@ class FzDistrictMatch {
}
},0)
}
def addThirdPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['C'] && d['C']=="新罗"){
def cityMap = CommonDatautils.findThirdPartyPrice(d['B'], d['C'], d['D'], 202312, 202403)
def priceMap = cityMap.get(d['B']+"##"+d['C']+"##"+d['D'])
if(priceMap && priceMap.zhugezhaofang_esf_price){
d['J'] = priceMap.zhugezhaofang_esf_price
}
if(priceMap && priceMap.anjuke_esf_price){
d['K'] = priceMap.anjuke_esf_price
}
if(priceMap && priceMap.soufang_esf_price) {
d['L'] = priceMap.soufang_esf_price
}
if(priceMap && priceMap.lianjia_esf_price){
d['M'] = priceMap.lianjia_esf_price
}
if(priceMap && priceMap.cityhouse_esf_price){
d['N'] = priceMap.cityhouse_esf_price
}
if(priceMap && priceMap.zhongyuan_esf_price){
d['O'] = priceMap.zhongyuan_esf_price
}
if(priceMap && priceMap.xingfuli_esf_price){
d['P'] = priceMap.xingfuli_esf_price
}
if(priceMap && priceMap.tuboshi_esf_price){
d['Q'] = priceMap.tuboshi_esf_price
}
if(priceMap && priceMap.get("5i5j_esf_price")){
d['R'] = priceMap.get("5i5j_esf_price")
}
def sellMap = CommonDatautils.findHouses2(d['B'], d['C'], "sell", 20231001, 20240407, d['D'], null, d['E'])
if(sellMap && sellMap.avg_price){
if(sellMap.avg_price.size()>=3){
d['S'] = Math.round(sellMap.avg_price.sum() / sellMap.avg_price.size());
}
}
}
},0)
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -20,16 +20,16 @@ class AddAvgpriceBatch {
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static {
citys = ["ÐìÖÝ"]//CommonDatautils.findAllCity()
citys = ["Ò˲ý"]//CommonDatautils.findAllCity()
println "加载城市完成:"+citys.size()
}
static main(args) {
def task = new AddAvgpriceBatch();
// task.run1();
task.run1();
// task.test()
// task.sendPriceCommand()
task.sendNewDistPrice()
// task.sendNewDistPrice()
}
def sendNewDistPrice(){
@@ -107,7 +107,7 @@ class AddAvgpriceBatch {
println "------------------------------------------------------------"
println "${city}小区量:"+districtKeysMap.size()
if(districtKeysMap){
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26"))
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2024-04-07"))
println "【2023-03-26】共计均价小区:"+lastWeekendStats.size()
//库中小区没有计算过均价的小区
def districtKeys = districtKeysMap.keySet();
@@ -118,7 +118,7 @@ class AddAvgpriceBatch {
def array = disKey.split("##");
def estateType2 = districtKeysMap.get(disKey)
//发重算均价命令
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd'))
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2024-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2024-04-07','yyyy-MM-dd'))
}
}
}
@@ -1,5 +1,7 @@
package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.poi.entity.Community
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -12,7 +14,41 @@ class CitySellStatTest {
static main(args) {
def task = new CitySellStatTest();
task.run()
// task.run()
task.cityPriceRun()
}
def cityPriceRun(){
def cityObjs = CommonDatautils.findCitys();
def rets = []
rets << ["序号","城市","城市级别","城市均价起始月","起始均价","城市均价最新月","最新均价"]
def index = 1
for(def o : cityObjs){
def ret = []
ret << index
ret << o.display
ret << o.new_city_grade
def row1 = findCityMonthPrice(o.display, 1)
if(row1){
ret << row1.date
ret << row1.avg_price
}else{
ret << ""
ret << ""
}
def row2 = findCityMonthPrice(o.display, -1)
if(row2){
ret << row2.date
ret << row2.avg_price
}else{
ret << ""
ret << ""
}
rets << ret
index++
}
ExcelUtils.write("D:\\数据\\其它\\全国城市均价.xlsx", rets, "城市均价")
}
def run(){
@@ -25,18 +61,20 @@ class CitySellStatTest {
},0)
}
def findFirstMonth(def city){
def findCityMonthPrice(def city, def asc){
def q = new BasicDBObject();
q.put("city", city)
def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("_id",1)).limit(1)
def month;
def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("date", asc)).limit(1)
def obj = null
for(def o : dbcursor){
month = o.date
obj = o
}
return month
return obj
}
def findTotalMonth(def city, def month){
def q = new BasicDBObject();
q.put("city", city)
@@ -23,18 +23,22 @@ class EsSearchUtils {
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
def rets = []
ElasticUtils.executeSqlQuery(sql,{ data->
def m = [:]
m.city = data.cityname
m.region = data.adname
m.name = data.name
m.address = data.address
m.type = data.type
m.code = data.typecode
def mapName = data.name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
def newName = name.replaceAll('[·\\(\\)()]|·', '')
if(newName == mapName){
def m = [:]
m.city = data.cityname
m.region = data.adname
m.name = data.name
m.address = data.address
m.type = data.type
m.code = data.typecode
def location = data.location
m.lng = location.lon
m.lat = location.lat
rets << m
def location = data.location
m.lng = location.lon
m.lat = location.lat
rets << m
}
})
return rets;
@@ -238,6 +238,7 @@ class FiledStrand {
}
static def getValue(def val){
val = val.replace(",","")
val = getNumber(val);
if(val){
if(val.indexOf(".")>-1 && !val.toString().endsWith(".0")){
@@ -0,0 +1,79 @@
package com.fangjia.script.gj
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class GreenBuild {
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new GreenBuild()
task.run()
}
def run(){
def datas = findAll()
def rets = match(datas)
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
}
def match(datas){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
data.match_id = resultMap.get("id").toString()
data.match_city = resultMap.get("city")
data.match_district = resultMap.get("district")
data.match_name = resultMap.get("name")
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
}else{
rets << [data._id.toString(), data.city, data.district, data.name]
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println "查询完成......."
latch.await();
return rets
}
def findAll(){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
}
}
@@ -13,12 +13,18 @@ import net.sf.json.groovy.JsonSlurper
import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
}
if(suggestName){
url = url +"&suggestName="+suggestName
}
if(suggestDistrict){
url = url +"&suggestDistrict="+suggestDistrict
}
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
@@ -12,8 +12,8 @@ class OpenUserBatch {
def task = new OpenUserBatch();
// task.run();
// task.test();
// task.addShenZhenFjtUser();
task.addFujianFjtUser()
task.addShenZhenFjtUser();
// task.addFujianFjtUser()
}
def test(){
@@ -46,20 +46,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252608 + i;
def id = 252893 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
onLineSqlInstance.execute(sql3)
}
}
@@ -52,7 +52,7 @@ class YouDaDataParser {
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -31,7 +31,34 @@ class CityStatExport {
// task.regionStat()
// task.stat()
// task.houseStat()
task.addFieldStat()
// task.addFieldStat()
task.exportRegionAvgPrice()
}
def exportRegionAvgPrice(){
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"];
def add = -9;
def months = []
while(add<=0){
months << DateUtils.addMonth(new Date(), add)
add++
}
def rets = []
rets << months
for(def region : regions){
def row = []
row << region
for(def i=0; i<months.size()-1;i++){
def start = months[i].replace("-","").toInteger()
def end = months[i+1].replace("-","").toInteger()
def avgprice = CommonDatautils.findHouseAvgPrice("福州", region, start, end)
row << avgprice
}
rets.add(row)
}
ExcelUtils.write("D:\\数据\\福州\\福州区域及城市走势_2024-03.xlsx", rets, "房源均价")
}
def addFieldStat(){
@@ -33,7 +33,16 @@ class CityStat {
// task.run6()
// task.run7()
// task.run8()
task.dealPriceStat()
// task.dealPriceStat()
task.countDist()
}
def countDist(){
ExcelUtils.modify("D:\\数据\\其它\\小区统计.xlsx", { d ->
if(d['B']){
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city", "region", "name"])
d['D'] = districts.size()
}
},0);
}
def dealPriceStat(){
@@ -0,0 +1,112 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
//郑州银行数据导出
class ZhengzhouBlankStat {
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
public static final def DISTRICT_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
]
static main(args) {
def task = new ZhengzhouBlankStat();
task.run()
}
def run(){
def citys = ["郑州","洛阳","平顶山","安阳","许昌","南阳","新乡","焦作","开封","信阳","驻马店","漯河","鹤壁","周口","濮阳","三门峡","商丘","济源"]
//statRegionDistrictCount(citys)
// statCityProcent(citys)
statCityHouseloudongProcent(citys)
}
def statRegionDistrictCount(citys){
def rets = []
rets << ["城市", "区县", "楼盘数量"]
for(def c : citys){
def regions = CommonDatautils.getCityRegions(c)
for(def r : regions){
def count = CommonDatautils.findRegionDistrictCount("district", c, r)
def ret = []
ret << c
ret << r
ret << count
rets << ret
}
}
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省区县小区量统计")
}
def statCityProcent(citys){
def rets = []
rets << ["城市","数据量","地址","地址别名","别名","楼栋总数","户数","产权年限","产权性质","建筑结构","竣工时间","小区当前均价"]
for(def c : citys){
def dbo = CommonDatautils.findCityInfo(c)
if(dbo){
def detail = dbo.district_detail
def ret = []
ret << c
if(detail){
ret << dbo.existing_district_count//数据量
ret << detail.address*100// 地址
ret << detail.addrAlias*100// 地址别名
ret << detail.alias*100// 别名
ret << detail.loudongCount*100// 楼栋总数
ret << detail.householdCount*100// 户数
ret << detail.propertyLife*100// 产权年限
ret << detail.houseProperty*100// 产权性质
ret << detail.buildStructure*100// 建筑结构(按材料划分)
ret << detail.finishBuildingDate*100// 竣工时间
ret << dbo.price_coverage*100// 小区当前均价
}
rets << ret
}
}
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省城市字段覆盖率统计")
}
def statCityHouseloudongProcent(citys){
def rets = []
rets << ["城市","楼栋","单元","所在层数","房号","面积","套型","楼层"]
for(def c : citys){
def dbo = CommonDatautils.findCityInfo(c)
if(dbo){
def detail = dbo.house_loudong_detail
def ret = []
ret << c
if(detail){
ret << detail.house_num_rate*100//楼栋
ret << detail.unit_num_rate*100// 单元
ret << detail.room_num_rate*100// 所在层数
ret << detail.room_num_rate*100// 房号
ret << detail.area_rate*100// 面积
ret << detail.room_info_rate*100// 套型
}
def loudong_detail = dbo.loudong_detail
if(loudong_detail){
ret << loudong_detail.loudong_height_rate*100// 楼层
}
rets << ret
}
}
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省房产信息字段覆盖率统计")
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.jsoup.Jsoup
class BaiduBaikeTown {
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
static main(args) {
def task = new BaiduBaikeTown()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
if(!row['E'] && row['A']=="郑州经济技术开发区"){
def q = new BasicDBObject()
q.put("desc",new BasicDBObject('$ne', 'null'))
q.put("name",row['A'])
def d = dzb_COLL.findOne(q)
def townMap = parse(d.desc, d.name)
row['D'] = townMap.town_str1
row['F'] = townMap.town_str2
// def ret = []
// ret << ["名称", "基本信息", "目录", "全文"]
// for(def d : dbcursor){
// def townMap = parse(d.desc, d.name)
// if(townMap.town){
// println d.name+" "+townMap.town
// }
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
// }
}
})
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
}
def parse(desc, name){
def map = [:]
if(desc){
def doc = Jsoup.parse(desc)
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
def s = doc.toString()
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
def jsonObj = JSONObject.fromObject(dataStr)
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
def towns = []
if(jsonObj && jsonObj.catalog){
def catalogs = jsonObj.catalog
def index = null
for(def log : catalogs){
def curIndex = log.index
if(log.title=="行政区划"){
index = log.index
}
if(index && curIndex.toString().startsWith(index+"-")){
towns << log.title
}
}
}
map.town_str1 = str
if(town){
map.town = town
}else{
map.town = ""
}
if(towns){
map.towns = towns.join(",")
}else{
map.towns = ""
}
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
map.town_str2 = str2
return map
}
map.town_str1 = ""
map.town_str2 = ""
map.town = ""
map.towns = ""
return map
}
}
@@ -0,0 +1,281 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
// task.addDevAddress(path)
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
}
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
o.put("city", d['C'])
o.put("district", d['D'])
o.put("name", d['A'])
o.put("level", d['E'])
o.put("type", d['F'])
if(d['G']){
o.put("township", d['G'])
}
if(d['H']){
def arr = d['H'].split(",")
if(arr){
o.put("alias", arr)
}
}
def id = addMongoDB(o)
o.remove("_id")
o.id = id.toString()
o.remove("create_time")
addEs(o, "192.168.0.89", "development_zone")
addEs(o, "192.168.0.52", "development_zone")
d['I'] = "已完成"
}else{
println d['A'] +" 已存在"
}
}
})
}
def addMongoDB(o){
o.put("create_time", new Date())
development_zone_100_CONN.save(o)
DEVELOPMENT_ZONE_43_COLL.save(o)
return o._id
}
def addEs(map, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
def addDevAddress(path){
ExcelUtils.modify(path,{ d->
if(!d['D'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
if(d['B'] == "乡镇街道"){
q.put("township", d['C'])
}else if(d['B'] == "地址"){
q.put("address", d['C'])
}else if(d['B'] == "园区"){
q.put("park", d['C'])
}
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
if(!mappingObj){
def o = new BasicDBObject()
o.put("province", obj.province)
o.put("city", obj.city)
o.put("district", obj.district)
o.put("name", obj.name)
o.put("type", d['B'])
if(d['B'] == "乡镇街道"){
o.put("township", d['C'])
}else if(d['B'] == "地址"){
o.put("address", d['C'])
}else if(d['B'] == "园区"){
o.put("park", d['C'])
}
def date = new Date()
o.put("c_date", date)
o.put("version", date.getTime())
DEV_MAPPING_100_CONN.save(o)
// o.id = o._id.toString()
// o.remove("_id")
// o.remove("c_date")
// addEs(o, "192.168.0.89", "dev_mapping")
// try{
// addEs(o, "192.168.0.52", "dev_mapping")
// }catch(Exception e){
// println "52 es error: "+e.printStackTrace()
// }
d['D'] = "已完成"
}else{
d['D'] = "重复"
println d['A'] +" "+ d['C']+" 重复"
}
}else{
d['D'] = "不存在"
println d['A'] +" 不存在"
}
}
},1)
}
def updateHasGov(province, city, district, street, hasGov){
def q = new BasicDBObject()
q.put("province", province)
q.put("city", city)
q.put("district", district)
q.put("street", street)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
//修改43
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
//修改89 administrative_mapping
update89EsAdministrativeMapping(province, city, district, street, hasGov)
//修改89 administrative_mapping
update52EsAdministrativeMapping(province, city, district, street, hasGov)
}else{
println province+" "+city+" "+district+" "+street+" 没找到"
}
//修改100 community
//修改89 community
//修改52 community
}
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class DistrictAliasProcess {
static main(args) {
def task = new DistrictAliasProcess();
// task.run()
task.fenQiProcess()
}
def run(){
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d->
if(d['H']=="是"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else if(d['H']=="否"){
def aliasDist = d['F']
def arrs = aliasDist.split("-")
def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district")
if(obj){
def alias = obj.alias
def mapping_alias = obj.mapping_alias
def udata = new BasicDBObject()
if(alias && alias.contains(d['D'])){
alias.remove(d['D'])
udata.put("alias", alias)
}
if(mapping_alias && mapping_alias.contains(d['D'])){
mapping_alias.remove(d['D'])
udata.put("mapping_alias", mapping_alias)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
}
}
})
}
def fenQiProcess(){
def q = new BasicDBObject()
q.put("cat","district")
q.put("status", 0)
q.put("master_district_name", 'master')
def dbcursor = DbUtils.seaweed.find(q)
def rets = []
for(def d : dbcursor){
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
}
for(def r : rets){
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
def lastObj = null
for(def j : rets){
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
lastObj = j;
break
}
}
if(lastObj){
println r
def udata = new BasicDBObject()
udata.put("master_district_name","")
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -0,0 +1,496 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class GXDdistrictMatch {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def regionCodeDataMap = PgsqlCommUtils.findDistrictMap();
def fullCityMap = CommonDatautils.queryFullNameCity();
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
task.addOldMatch()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
// task.runMatch()
}
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ'] && d['I']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
d['AL'] = jsonObject.get("district")
d['AM'] = jsonObject.get("name")
d['AN'] = jsonObject.get("address")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['AO'] = dbo.status
d['AP'] = dbo.name_cat
d['AQ'] = dbo.estate_type2
}
}
}
});
}
def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
if(!d['U']){
def fullCityObj = fullCityMap.get(d['AD'])
if(fullCityObj){
def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
def obj = findPoiFromDB(fullCityObj.city, fullRegion, d['C'])
if(obj){
d['U'] = "地图库"
d['V'] = obj.city
d['W'] = obj.region
d['X'] = obj.name
d['Y'] = obj.type
d['Z'] = obj.address
d['AA'] = obj.lng
d['AB'] = obj.lat
}else{
obj = findDatas(d['AD'], d['AE'], d['C'], d['C'].replaceAll('[·\\(\\)()]|·', ''))
if(obj){
d['U'] = "楼盘抓取库"
d['V'] = obj.city
d['W'] = obj.region
d['X'] = obj.name
d['Y'] = obj.property_type
d['Z'] = obj.address
def loc = obj.location
if(loc){
d['AA'] = loc.x
d['AB'] = loc.y
}
}
}
}
}
})
}
def findPoiFromDB(city, region, name){
def rets = EsSearchUtils.findPois(city, region, name)
def obj = null
if(rets && rets.size()==1){
obj = rets[0]
}
return obj
}
def run(){
def fileName = "D:\\数据\\国信达\\接口匹配.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['V'] && d['A']){
def city = getCityFromCityCode(d['A'])
def region = regionCodeDataMap.get(d['A'] + "#" + d['E'])
if (city && region) {
region = RegionData.formatRegion(city, region)
d['AD'] = city
d['AE'] = region
if (region) {
d = match(city, region, d)
} else {
println d['A'] + "#" + d['E'] + " 区域无法格式化"
}
} else {
println d['A'] + "#" + d['E'] + " 无法格式化"
}
}
})
}
def runMatch(){
def fileName = "D:\\数据\\福州\\建行测试匹配.xlsx"//11城重新匹配.xlsx 福州贝壳数据.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['A']
println d['C']
if(d['B'] && d['A']){
def region = RegionData.formatRegion(city, d['B'])
if (region) {
d = match(city, region, d)
} else {
println d['A'] + "#" + d['E'] + " 区域无法格式化"
}
}
})
}
def updateDistStatus(){
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
def index = 0
ExcelUtils.modify(fileName, { d ->
if(d['U'] && ["名称匹配","名称匹配(Update)","名称在别名中","名称在别名中(Update)","别名匹配"].contains(d['U'])){
index++
def status = d['Z']
def name_cat = d['AA']
println d["V"]+" "+d["X"]+name_cat
if(status!=null && status.toString().toDouble().toInteger()==0){
if(name_cat==null){//&& name_cat.toString().toDouble().toInteger()!=0
def dist = CommonDatautils.findDistrict(d["V"],d["W"],d["X"], "district")
if(dist){
def delUdata = new BasicDBObject()
delUdata.put("name_cat", 0)
delUdata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
saveOptLog(dist.city, dist.region, dist.name, "update")
}
}
}
}
})
}
def restoreDist(){
def fileName = "D:\\数据\\国信达\\11城小区需恢复数据.xlsx"//国信达11城(全).xlsx
def index = 0
ExcelUtils.modify(fileName, { d ->
if(d['U']){
index++
def status = d['Z']
println d["V"]+" "+d["X"]
if(status && status.toString().toDouble().toInteger()==1){
def dists = CommonDatautils.findAllStatusDistrict(d["V"],d["W"],d["X"], "district")
if(dists && dists.size()==1){
def dist = dists[0]
def delUdata = new BasicDBObject()
delUdata.put("name_cat", 0)
delUdata.put("status", 0)
delUdata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
saveOptLog(dist.city, dist.region, dist.name, "update-status")
}
}
}
})
}
def saveToDB(){
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
ExcelUtils.modify(fileName, { d ->
if(d['A']){
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def alias = []
if(d['G'] && name!=d['G']){
alias << d['G']
}
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
// alias << d['AI']
// }
if(alias){
udata.put("alias", alias)
}
def addr = AddressUtil.formatAddress(d['A'], d['D'])
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
// if(d['AH']){
// type = d['AH']
// }else{
// type = d['Y']
// }
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
if(d['F']){
Long lng = d['F']*10000000000
udata.put("lng", lng)
}
if(d['E']){
Long lat = d['E'] * 10000000000
udata.put("lat", lat)
}
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
def opt = "add"
if(s && s=='成功'){
saveOptLog(udata.city, udata.region, udata.name, opt)
}
}
})
}
def match(city, region, d){
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
if(!d['U']){
d = matching(city, region, d, d['C'].replace("小区","").replace("号楼","号院").replace("号","号院"), d['D'], "(Update)")
if(!d['U']){
d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)")
}
if(!d['U']){
if(d['F'] && d['G']){
d = matchAddr(city, region, d, "")
}
}
if(!d['U']){
d = matchApi(city, region, d, "property", false)
if(!d['U']){
d = matchApi(city, region, d, null, false)
}
}
if(!d['U']){
d = matching2(city, region, d, d['C'], d['D'], "")
}
if(!d['U']){
if(d['C'].contains("小区")){
d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)")
}else{
d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)")
}
}
}
return d
}
def matchApi(city, region, d, cat, useMap){
def jsonObject = null
if(!jsonObject && d['H']){
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
}
if(!jsonObject && d['C']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
if(!jsonObject && d['D']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
}
}
if(jsonObject != null) {
d['U'] = "接口匹配"
d['V'] = jsonObject.get("city")
d['W'] = jsonObject.get("district")
d['X'] = jsonObject.get("name")
def type = jsonObject.get("category")//propertyType
if(type && type!='null'){
d['Y'] = type
}
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['Z'] = dbo.status
d['AA'] = dbo.name_cat
}
d['AC'] = jsonObject.get("credit")
if(jsonObject.get("address") && jsonObject.get("address")!='null'){
d['AH'] = jsonObject.get("address")
}
}
return d
}
def matching(city, region, d, name, alias, str){
//正常状态下有没有
def rets = CommonDatautils.findDistrictList(city, region, name, "district")
if(rets){
if(rets.size()>=1){
d = setRowValue(rets[0], d, "名称匹配"+str)
}
}else{
rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if(rets && rets.size()>=1){
d = setRowValue(rets[0], d, "名称在别名中"+str)
}else{
rets = CommonDatautils.findDistrictByAddress(city, region, name, "district")
if(rets && rets.size()>=1){
d = setRowValue(rets[0], d, "名称在地址中"+str)
}else{
if (d['D']) {
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
if (rets) {
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "别名匹配"+str)
}
}
}
}
}
}
if(rets){
d['AI'] = rets.size()
}
return d;
}
def matching2(city, region, d, name, alias, str){
def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的
if(rets){
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "名称匹配(非正常)"+str)
}
}else{
rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district")
if (rets && rets.size() >= 1) {
d = setRowValue(rets[0], d, "别名匹配(非正常)"+str)
}else{
rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district")
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "别名匹配(非正常)"+str)
}
}
}
if(rets){
d['AI'] = rets.size()
}
return d;
}
def matchAddr(city, region, d, str){
def rets = CommonDatautils.findDistrictByAddress(city, region, d['F']+d['G'],"district")
if(rets){
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "地址匹配")
}
}
if(rets){
d['AI'] = rets.size()
}
return d
}
def setRowValue(dbo, d, remark){
if (dbo) {
d['V'] = dbo.city
d['W'] = dbo.region
d['X'] = dbo.name
d['Y'] = dbo.estate_type2
d['Z'] = dbo.status
d['AA'] = dbo.name_cat
d['U'] = remark
d['AH'] = dbo.address
}
return d;
}
def getCityFromCityCode(cityCode){
def city = null
for(def m : cityCodeDataMap){
if(m.value == cityCode){
city = m.key
break;
}
}
return city;
}
def filterSpacialChar(def s){
def pattern = ~/[()!;\·]\[【】()]/
def matcher = pattern.matcher(s)
def count = matcher.getCount()
if(count && count>0){
return matcher.replaceAll('_')
}
return null;
}
def findDatas(def city, def region, def name1, def name2){
def sql = "select * from community where city='${city}' and region='${region}' and (name='${name1}' or name='${name2}')"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
println d.property_type
})
if(rets){
def obj = null
for(def r : rets){
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
obj = r
break
}
}
if(obj){
return obj;
}else{
return rets[0];
}
}
return null;
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","district");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", type)
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -0,0 +1,372 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* 名录集
*/
class MinglujiData {
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL)
static main(args) {
def task = new MinglujiData()
task.run()
}
def run(){
def prov = "浙江省"
def city = "嘉兴"
// exportDatas(city)
matchDev(prov, city)
matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
fiveLevelSplit(city)
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
}
def matchDev(prov, city){
def devs = loadAllDevs(prov)
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName, { d ->
if (d['D']) {
def street = d['D']
def splitStreet = ""
if (street) {
if (street.contains(d['C'])) {
def start = street.toString().lastIndexOf(d['C']) + d['C'].size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
} else {
splitStreet = street
}
}
d['K'] = splitStreet
}
if (d['I']) {
def splitCommunity = ""
if (d['C'] && d['I'].toString().contains(d['C'])) {
splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size())
} else {
splitCommunity = d['I']
}
d['L'] = splitCommunity
}
if (d['E'] && !d['N']) {
def obj = findDevByName(d['E'])
if (!obj) {
obj = findDevByAlias(d['E'])
if (!obj) {
obj = findDebBySimlar(devs, d['E'], d['B'])
if (!obj) {
obj = findDevByPark(d['A'], d['B'], d['E'])
if (!obj) {
} else {
d['N'] = "园区"
}
} else {
d['N'] = "相似"
}
} else {
d['N'] = "别名"
}
} else {
d['N'] = "相同"
}
if (obj) {
d['M'] = obj.name
d['O'] = obj.city
d['P'] = obj.district
}
}
})
}
i++
println fileName +" 完成(匹配开发区不使用接口)"
}
}
def tempMatchDev(prov){
def devs = loadAllDevs(prov)
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d->
if(d['U'] && !d['W'] ){
def obj = findDevByName(d['U'])
if(!obj){
obj = findDevByAlias(d['U'])
if(!obj){
obj = findDebBySimlar(devs, d['U'], d['B'])
if(!obj){
obj = findDevByPark(d['A'], d['B'], d['U'])
if (!obj) {
}else{
d['X'] = "园区"
}
}else{
d['X'] = "相似"
}
}else{
d['X'] = "别名"
}
}else{
d['X'] = "相同"
}
if(obj){
d['W'] = obj.name
d['Y'] = obj.city
d['Z'] = obj.district
}
}
})
}
def matchCommunity(city){
def i = 1;
while(i<20){
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['A'] && d['B'] && d['L'] && !d['Q']){
def list = findCommunityByName(d['A'], d['B'], d['L'])
if(!list){
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
if(!list){
}else{
d['T'] = "包含"
}
}else{
d['T'] = "相同"
}
if(list){
if(list.size()==1){
def obj = list[0]
d['Q'] = obj.name
d['R'] = obj.street_name
d['U'] = obj.area_name
d['V'] = obj.city_name
}else if(list.size()>=2){
for(def l : list){
if(d['C']==l.area_name){
d['Q'] = l.name
d['R'] = l.street_name
d['U'] = l.area_name
d['V'] = l.city_name
break;
}
}
d['S'] = list.size()
}
}
}
}, 0)
println fileName +" 运行完成"
}
i++
}
}
def fiveLevelSplit(city){
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
}
File file = new File(fileName);
if (file.exists()) {
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['J'] && !d['X']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['J'])
if(jsonObject) {
if (jsonObject.get("address") && jsonObject.get("address") != "null") {
d['X'] = jsonObject.get("address")
}
}
}
})
}
i++
}
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
def obj = new BasicDBObject()
obj.put("province", d['A'])
obj.put("city", d['B'])
obj.put("district", d['C'])
obj.put("name", d['D'])
obj.put("address", d['E'])
obj.put("dev_name", d['F'])
obj.put("committee", d['G'])
obj.put("township", d['H'])
company_100_CONN.save(obj)
})
}
def loadAllDevs(prov){
def q = new BasicDBObject()
q.put("province", prov)
return development_zone_CONN.find(q).toList()
}
def findDebBySimlar(def devs, def name, def city){
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
def last = null
for(def d : devs){
def alias1 = null
if(!d.alias){
alias1 = []
}else{
alias1 = d.alias
}
if(!alias1.contains(d.name)){
alias1 << d.name
}
if(alias1){
try {
for(def n : alias1){
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
last = d
break
}
}
}catch(exception){
println alias1
}
}
if(last){
break
}
}
return last
}
def findDevByName(name){
def q = new BasicDBObject()
q.put("name", name)
return development_zone_CONN.findOne(q)
}
def findDevByAlias(name){
def q = new BasicDBObject()
q.put('alias', new BasicDBObject('$in',[name]))
return development_zone_CONN.findOne(q)
}
def findDevBySimlar(name){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
return development_zone_CONN.findOne(q)
}
def findDevByPark(prov, city, name){
def q = new BasicDBObject()
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("type", "园区")
return dev_mapping_CONN.findOne(q)
}
def findCommunityByName(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", name)
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findCommunityBySimlar(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findMinglujiDatas(city){
def keys = new BasicDBObject()
keys.put("c_address",1)
keys.put("c_city",1)
keys.put("c_committee",1)
keys.put("c_community",1)
keys.put("c_company",1)
keys.put("c_district",1)
keys.put("c_street",1)
keys.put("kind",1)
keys.put("name",1)
keys.put("province",1)
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
def rets = []
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
for(def d : datas){
def street = d.c_street
def splitStreet = ""
if(street){
if(street.contains(d.c_district)){
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
}
}
def splitCommunity = ""
if(d.c_committee){
if(d.c_committee.contains(d.c_district)){
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
}
}
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
}
return rets
}
}
@@ -0,0 +1,423 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
static main(args) {
def task = new PoiLocation();
// task.run();
task.regionAndStreetProcess()
}
def run(){
def citys = [""]
citys = citys.reverse()
for(def city : citys){
findPois(city)
}
}
def regionAndStreetProcess(){
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
for(def city : citys){
def datas = findAllPoisByCity(city)
process(datas)
}
}
def findAllPoisByCity(city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findPois(city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("run_status", new BasicDBObject('$ne', 1))
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
def index = 0
for(def d : dbcursor){
rets << d
if(rets.size()>=10000){
index++
def datas = checkData(index, rets)
// updateToEs(datas)
// updateToEs2(datas)
// updateToMG(datas)
rets = []
print "."
}
}
if(rets.size()>0){
index++
def datas = checkData(index, rets)
// updateToEs(datas)
// updateToEs2(datas)
// updateToMG(datas)
rets = []
print "."
}
return rets;
}
def process(datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
dataRun(d)
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = running(d)
rets << map
if(map.update_status && map.update_status!=100){
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
}
if (map.update_status && map.update_status !=100) {
updateMapPoi(map.id, map.township)
}
updateToMG(map)
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def running(d){
def regions = getFullRegion(d)
def town = null
def map =[:]
if(d.location){
//判断区域否在围栏里
def regionFlag = null
if(d.district){
for(def rs : regions){
if(rs.shape && rs.shape.coordinates){
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
}
if(regionFlag){
//判断township是否在统计局中
if(d.township){
town = getSimilarTown(regions, d.city, d.township)
}
if(town){
if(town.street == d.township){
//打标记
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = town.street
}
}else{
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
if (mapTown) {
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = mapTown.street
}else{
if(d.district!=gaodeMap.district){
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
}
}
}else{
//打标记“经纬度有误,不在区域围栏中”
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
return map;
}
def dataRun(d){
def regions = getFullRegion(d)
if(d.location) {
//判断区域否在围栏里
def regionFlag = null
if (d.district) {
for (def rs : regions) {
if (rs.shape && rs.shape.coordinates) {
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
}
if (regionFlag) {//在围栏里
//判断township是否在统计局中
// if(d.township){
// town = getSimilarTown(regions, d.city, d.township)
// }
}else{//不在转栏里,重新搜索
println(d)
}
}
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.district)){
names << r
}
}
}
return names;
}
def getSimilarTown(regions, city, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(city+"#"+r.district)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if (map.update_status && map.update_status !=100) {
updateMapPoi(map.id, map.township)
}
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.update_status && map.update_status==2){
map.remove("run_status")
map.remove("update_status")
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
def updateToMG(m){
// def rets = []
// for(def i=0;i<rows.size();i++){
// def m = rows[i]
def udata = new BasicDBObject()
m.each{k,v->
if(k!="id"){
udata.put(k,v)
}
}
def q = new BasicDBObject()
q.put("_id", m.id)
if(udata){
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
}else{
println m.toString()+" 无法修改-无值"
}
// }
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class TempExportData {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new TempExportData()
task.run()
}
def run() {
def citys = CommonDatautils.findAllCity()
def rets = []
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
for (def city : citys) {
def cityRets = getResults(city)
rets.addAll(cityRets)
}
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
}
def getResults(city) {
def rets = []
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
final CountDownLatch latch = new CountDownLatch(dists.size())
final AtomicInteger count_ = new AtomicInteger(0)
for (def i = 0; i < dists.size(); i++) {
def d = dists[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.addAndGet(1)
try {
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
if (objs) {
def ret = []
ret << d._id.toString()
ret << d.city
ret << d.region
ret << d.name
ret << d.address
def str = ""
for (def o : objs) {
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
}
ret << str
ret << objs.size()
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
println(city +" "+rets.size())
return rets;
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
}
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
}
@@ -0,0 +1,159 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@BatchOwner(Watcher.GaoJuan)
class ThardParthAnalyBatch extends BaseBatch{
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThardParthAnalyBatch(), args)
}
// static main(args) {
// def task = new ThardParthAnalyBatch()
// task.run()
// }
@Override
void process(Object args) {
run()
}
def run(){
def fjwTotalCount = CommonDatautils.findAllCount("district")
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
println "统计月份:"+sDate+" 查询更新时间:"+mdate
def crawlTotalCount = findAllEsDataCount(mdate)
println "抓取第三方小区总量:"+crawlTotalCount
def synchTotalCount = findAllMongoDatasCount(sDate)
println "同步到第三方库小区总量: "+synchTotalCount
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
def synchDistCount = findAllMongoDistrictCount(sDate)
println "同步到第三方库有价格小区量: "+synchDistCount
def matchTotalCount = findMatchSuccessCount(sDate)
println "第三方匹配成功总量:"+matchTotalCount
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
println "第三方匹配成功小区量:"+matchDistTotalCount
def precent = matchDistTotalCount*100.0/fjwTotalCount
precent = DataHandleUtils.getDoubleRound(2, precent)
println "匹配成功占比:"+precent+"%"
def noMatchTotalCount = findNoMatchCount(sDate)
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
}
def findAllEsDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllEsDistDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllMongoDatasCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
return thirdPartyColl.count(q)
}
def findAllMongoDistrictCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
q.put("avg_price", new BasicDBObject('$gt', 0))
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("region",1)
keys.put("name",1)
def dbcursor = thirdPartyColl.find(q,keys)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findAllMongoDatasHavePricesCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
return thirdPartyColl.count(q)
}
def findMatchSuccessCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
return thirdPartyColl.count(q)
}
def findMatchSuccessDistCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
def dbcursor = thirdPartyColl.find(q)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findNoMatchCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
return thirdPartyColl.count(q)
}
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
def udata = new BasicDBObject()
udata.put("fjw_dist_total", fjwTotalCount)
udata.put("crawl_total", crawlTotalCount)
udata.put("synch_total", synchTotalCount)
udata.put("synch_have_price_total", synchHavePriceTotalCount)
udata.put("synch_have_price_district_total", synchDistCount)
udata.put("match_total", matchTotalCount)
udata.put("match_dist_total", matchDistTotalCount)
udata.put("match_dist_rate", precent)
udata.put("no_match_total", noMatchTotalCount)
udata.put("c_date", new Date())
if(obj){
udata.put("m_date", new Date())
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
}else{
udata.put("s_date", sDate)
THIRD_PARTY_STAT.save(udata)
}
}
}
@@ -0,0 +1,51 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.regex.Pattern
class TownshipPrcoess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
static main(args) {
def task = new TownshipPrcoess();
task.run()
}
def run() {
def fileName = "D:\\数据\\数据建设\\乡镇别名问题.xlsx"
ExcelUtils.modify(fileName, { d ->
if(!d['G']){
// def name1 = d['E'].replace("街道","").replace("镇","").replace("乡","")
// def name2 = d['F'].replace("街道","").replace("镇","").replace("乡","")
// if(name1 == name2){
// d['G'] = "不处理"
// }
def lists1 = findCommunityTown(community_CONN, d['C'], d['D'], d['E'])
def lists2 = findCommunityTown(community_CONN, d['C'], d['D'], d['F'])
if(lists1 && lists2){
d['G'] = "需删除"
}
}
})
}
def findCommunityTown(community_CONN, city, region, town){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
q.put("street_name", town)
def rets = community_CONN.find(q).toList()
return rets;
}
}
@@ -0,0 +1,632 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class TownshipStrandUpdate {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def fullCityMap = null
def regionMap = null
def commMap = null
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
static main(args) {
def task = new TownshipStrandUpdate()
// task.process()
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// task.updateSpecCityTown()
}
void process() {
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
citys = citys.reverse()
for(def city : citys){
//
// if(["中山","东莞"].contains(city)){
// continue
// }
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
// if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
// }
}
}
// 相同且为统计局乡镇
// 更正
// 不同且不为统计局乡镇
// 区域不同
// 地图无乡镇街道
// 城市不同
def stat(){
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
def rets = []
rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
def map = [:]
def datas = null
if(file.exists()){
datas = ExcelUtils.read(fileName,0, 1)
for(def d : datas){
def key = "空值"
if(d['S']){
key = d['S']
}
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
}
println city
map.each {k,v->
println k+" "+v
}
println "--------------------------------------------------------------------------------------------"
if(map){
rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")]
}
}
ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺")
}
def updateTown(){
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(file.exists()){
println fileName
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['S']!='相同且为统计局乡镇' && d['S']){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['B'])
q.put("name", d['C'])
q.put("status", 0)
q.put("cat","shop")
def udata = new BasicDBObject()
udata.put("township", d['R'])
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
})
}
}
}
def checkData(def dists){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets << running(d)
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return dists
}
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
regions = []
regions << fullCity.city
}
def gaodeMap = null
def town = null
def mapTown = null
def oldSimlarTown = null
def newSimlarTown = null
if(d.lng){
d.lng = d.lng/10000000000.0
d.lat = d.lat/10000000000.0
if(d.township){
town = getSimilarTown(regions, fullCity, d.township)
}
if(town && town.shape && town.shape.coordinates){
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
d.polygon_town = town.street
}
}
d.strand_street = ""
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
if(mapTown){
d.strand_street = mapTown.street
if(mapTown.shape && mapTown.shape.coordinates){
d.map_town_shape = "有"
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
if(flag){
d.map_polygon_town = mapTown.street
}
}
}else{
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
if(oldSimlarTown){
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
d.old_similar_town_score = oldSimlarTown.score
}
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
if(newSimlarTown){
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
d.new_similar_town_score = newSimlarTown.score
}
}
}
}
def mapCity = null
def mapRegion = null
def mapTownship = null
def mapAddr = null
if(gaodeMap){
if(gaodeMap.city && gaodeMap.city!="[]"){
mapCity = gaodeMap.city
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}else{
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
mapRegion = mapCity
}
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
}
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
mapAddr = gaodeMap.format_addr
}
}
d.mapCity = mapCity
d.mapRegion = mapRegion
d.mapTownship = mapTownship
d.mapAddr = mapAddr
if(mapCity && mapRegion){
if(mapCity == fullCity.city){
if(regions.contains(mapRegion)){
if(mapTownship){
if(town){
d.old_tjj_township = "是"
}else{
d.old_tjj_township = "否"
}
if(mapTown){
d.new_tjj_township = "是"
}else{
d.new_tjj_township = "否"
}
if(mapTownship == d.township){
if(town){
d.remark = "相同且为统计局乡镇"
}else{
d.remark = "非统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}else{
if(mapTown){
d.remark = "更正"
}else{
d.remark = "不同且不为统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}
}else{
d.remark = "地图无乡镇街道"
d.opt = "地图无街道信息"
}
}else{
d.remark = "区域不同"
d.opt = "经纬度可能跨区域"
}
}else{
d.remark = "城市不同"
d.opt = "经纬度可能跨城市"
}
}
if(d.map_town_shape && d.map_town_shape=="有"){
if(!d.map_polygon_town){
d.opt = "经纬度可疑"
}
}
return d;
}
def getSimilarTownByStr(fullCity, townstr, townList){
def maxScore = 0
def town = null
for(def townObj : townList){
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
if(score > maxScore){
maxScore = score
town = townObj
}
}
if(town){
town.score = maxScore
}
return town
}
def getSimilarTown(regions, fullCity, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(fullCity.city+"#"+r)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def getFullRegion(fullCity, d){
def regions = regionMap.get(fullCity.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
names << r.district
}
}
}
return names;
}
def createDatas(dists){
def rets = []
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
for(def d : dists){
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
}
return rets;
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(township || district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
d['AA'] = lists.size()
//只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
def newRegion = RegionData.formatRegion(d['B'], d['M'])
if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0]
def arrs = d['X'].split(" ")
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
d['AB'] = "已处理"
}
}
}
})
}
def cleanTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['AC']){
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
if(dbo && dbo.township){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
}
}
})
}
def checkTownFromMongo(){
def citys = CommonDatautils.findAllCity()
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
citys = citys.reverse()
def index = 0
for(def city : citys) {
index++
if (["中山", "东莞"].contains(city)) {
continue
}
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
for (def d : dists) {
if (d.township) {
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def town = getSimilarTown(regions, fullCity, d.township)
if (!town) {
town = findStreet(d.city, d.city, d.township)
if (!town) {
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
println d.city + " " + d.region + " " + d.township + " " + d.name
}
}
}
}
println index + " " + city + " 完成"
}
}
def findStreet(city, district, street){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
q.put("city", new BasicDBObject('$regex', provPattern))
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
if(list){
if(list.size()==1){
return list[0]
}else{
println city+ " "+district+" "+street+" 有多个"
}
}
return null
}
def updateTownMapping(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['F']){
// def udata = new BasicDBObject()
// udata.put("mzb_district_name", d['F'])
//
// def updateQuery = new BasicDBObject()
// updateQuery.put("city", d['C'])
// updateQuery.put("district", d['D'])
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
def city = RegionData.formatCity(d['C'])
if(city){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def updateQuery = new BasicDBObject()
updateQuery.put("category", "region")
updateQuery.put("city", city)
updateQuery.put("name", region)
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
if(list && list.size()==1){
def dbo = list[0]
def query = new BasicDBObject()
query.put("_id", dbo._id)
// def pattern = dbo.pattern
// if(pattern.contains(d['F'])){
// pattern.remove(d['F'])
// }
def udata = new BasicDBObject()
udata.put("org_name", d['F'])
// udata.put("pattern",pattern)
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
}else{
println city +" "+region+" "+list.size()
}
}
}
}
})
}
def updateDevMapping(){
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("district", d['B'])
q.put("name",d['C'])
q.put("type", "乡镇街道")
q.put("township", d['D'])
def list = dev_MAPPING_COLL_100.find(q).toList()
if(list && list.size()==2){
def obj = list[0]
def removeQuery = new BasicDBObject()
removeQuery.put("_id", obj._id)
dev_MAPPING_COLL_100.remove(removeQuery)
}else{
println q
}
})
}
def updateSpecCityTown(){
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
// def citys = ["中山"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
ExcelUtils.modify(fileName,{d->
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
if(list.size()>1){
println d['A'] +" "+d['C'] + " 有多个"
}else if(list.size()==1){
def dbo = list[0]
def town = null
// if(d['X']){
// town = d['X']
// }else
if(d['R']){
town = d['R']
}
if(!town && d['M']){
town = d['M']
}
def region = RegionData.formatRegion(d['A'], town)
if(region){
def q = new BasicDBObject()
q.put("region", region)
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", town)
udata.put("region", region)
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
}else{
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
}
}
})
}
}
}
@@ -0,0 +1,46 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class TradeDatasDistBatch {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
static main(args) {
def task = new TradeDatasDistBatch()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
udata.put("name", d['C'])
udata.put("alias", d['D'])
alias_fang_COLL.save(udata)
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def flag = null
// if(!dist){
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }else{
// flag = "名称匹配"
// }
// if(flag){
// d['F'] = dist.city
// d['G'] = dist.region
// d['H'] = dist.name
// }
})
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -135,6 +136,26 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def format(def city, def address){
def jsonObj = null
def content = null
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -228,9 +249,9 @@ class FangjiaClientUtils {
def url = null;
try {
address = address.replace("{","").replace("\\","")
url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -314,13 +335,13 @@ class FangjiaClientUtils {
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
@@ -332,18 +353,27 @@ class FangjiaClientUtils {
public static def formatNinelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
def url = null
address = address.replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
if (content) {
break
}
retryCount++
}
if (!content) {
return null
}
try {
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
println address +" "+content+" "+e.getMessage()
}
return jsonObj;
}
@@ -36,18 +36,16 @@ class SeaweedDataMatchProcess {
}
}
if(!rMap && name){
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
}
if(!rMap && address){
if(!rMap && address && name){
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
rMap = OpenApiUtils.matchDistrict(city, address, region, true, name)
}
}
if(!rMap && name){
rMap = OpenApiUtils.matchDistrict(city, name, region, false, null)
}
// if(rMap && rMap.id && region.contains(rMap.region)) {
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
// if (obj) {
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.es
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -16,7 +17,13 @@ class EsSelectTest {
static main(args) {
def task = new EsSelectTest()
task.run()
// task.run()
task.test()
}
def test(){
def ret = CommonDatautils.findLogs("1007741", "/property/evaluate", "2024-03-01", "2024-03-28")
}
def run(){
@@ -0,0 +1,138 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 房价网内部模型估值项目
* 小区建筑类型数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataCommbuildtypeBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distIds = FjwPgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
def rows = createDatas(cityCodeDatas.get(city), distIds)
rets.addAll(rows)
println city +" "+rows.size()
}
delPgData()
saveToPgSql(rets)
}
def createDatas(cityCode, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def id = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dist = CommonDatautils.findDistrictFieldById(id, ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
if(dist){
def ret = createData(cityCode, dist)
if(ret){
rets.addAll(ret)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(cityCode, dist){
def types = CcbCodeUtils.parseBuildType(dist.build_type)
def rets = []
for(def type : types){
def ret = []
ret << cityCode
ret << dist._id.toString()
ret << type
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
rets << ret
}
return rets;
}
def delPgData(){
sqlInstance.execute("delete from base.bigdata_commbuildtype")
println "base.bigdata_commbuildtype 数据已删除"
}
def saveToPgSql(rows){
def sqls = []
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
def sql = "insert into base.bigdata_commbuildtype values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
println "start: "+new Date()
def index =0
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
index++
println index
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,372 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 房价网内部估值项目使用
* 小区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
def fangjiaDists = new HashSet()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
for(def d : dists){
fangjiaDists << d._id.toString()
}
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
}
// if(today > 10){
delPgData()
saveToPgSql(rows)
// }
}
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
if(!fangjiaDists.contains(ccbId)){
def datas = ccbMaps.get(ccbId)
datas[datas.size()-1] = 1
rets << datas
}
}
println "添加删除状态小区:"+rets.size()
return rets
}
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
//def rets = Collections.synchronizedList(new ArrayList());
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDataMap, distCodeMap, dist)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(def cityCodeDataMap, def distCodeMap, def dist){
def ret = []
try{
def data = distCodeMap.get(dist.region)
if(!data){
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
return ret;
}
ret << cityCodeDataMap.get(dist.city)
if(data){
ret << data.district_code
}else{
ret << ""
}
ret << dist._id.toString()
if(data){
ret << data.district
}else{
ret << ""
}
ret << dist.name
ret << dist.address
if(dist.lat){
def lat = dist.lat/10000000000d
def lng = dist.lng/10000000000d
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << dist.all_area
ret << dist.build_area
ret << dist.virescence_percent
ret << dist.cubage_percent
ret << dist.car_site_count
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "pa101"
ret << ""
if(dist.address){
if(dist.street){
ret << dist.street
if(dist.street_number){
ret << dist.street_number
}else{
ret << ""
}
}else{
ret << ""
ret << ""
// def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
// if(jsonObj && jsonObj.result && jsonObj.result.street){
// if(jsonObj.result.street){
// ret << jsonObj.result.street
// }else{
// ret << ""
// }
//
// if(jsonObj.result.streetNumber){
// ret << jsonObj.result.streetNumber
// }else{
// ret << ""
// }
// }else{
// ret << ""
// ret << ""
// }
}
}else{
ret << ""
ret << ""
}
ret << 0
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << ""
ret << ""
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name+" error"
e.printStackTrace()
}
return ret;
}
public def findPgCcbDistrict(cityid){
def sql = "select * from base.bigdata_community where urbn_id='${cityid}'";
def map = [:]
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_dtl_adr
data << r.gujia_bds_ltt
data << r.gujia_bds_lgt
data << r.gujiabdsocpoldtotarea
data << r.gujiabds_bld_tot_area
data << r.gujia_bds_grnrate
data << r.gujia_bds_vol_rate
data << r.gujia_bds_park_num
data << r.gujia_bds_bld_yr_rc
data << r.gujia_bds_hs_use_val
data << r.totunits
data << r.gujia_bds_data_tm_val
data << r.gujiabds_udt_flag_val
data << r.data_udt_tm
data << r.gujia_comm_tp_val
data << r.gujia_comm_fmnm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujiawthr_new_pan_ind
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.gujia_src_co
data << r.gujia_vlgtwn_str_mng
data << r.gujia_lgc_admndiv_cd
data << r.gujia_lgc_adiv
data << r.gujia_bds_bds_st
map.put(r.gujia_comm_id, data)
});
return map
}
def delPgData(){
sqlInstance.execute("delete from base.bigdata_community")
println "base.bigdata_community 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_community values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0
sqlInstance.withBatch(50000, sql){ ps->
for(def r : newRows) {
index++
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
index
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
sqlInstance.executeUpdate(sql)
}
}
}
@@ -0,0 +1,790 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.*
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 成交数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataDealBatch extends BaseBatch{
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataDealBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
println "txt 生成 " + rows.size()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas)
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
def result = Collections.synchronizedList(new ArrayList());
dealIds = Collections.synchronizedList(dealIds);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
// def lianjiaPriceMap = findThirdPartyPrice("上海")
// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
def ccbPriceMap = null
if(city == "上海"){
ccbPriceMap = [:]//getCommunityPrices("上海")
}
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(dealIds.contains(trade._id.toString())){
println "重复:"+trade._id.toString() + " "+ trade.city+ " " +trade.district_name
}else{
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}
}else{
def ret = converData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println "查询完成......."
latch.await();
return result
}
def getCommunityPrices(def city){
def sql = "select commid, avg_price from community_base_price where year_month=${endDate} and cityname='${city}'";
def map = [:]
try{
sqlInstance.eachRow(sql,{
map.put(it.commid, it.avg_price)
});
}catch(Exception e){
//getPpeGTE15Datas(city)
}
return map
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for (def db : dbcursor) {
def price = null
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
price = db.lianjia_esf_price
}
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
price = db.beike_esf_price
}
if(price){
map.put(db.region+"##"+db.name, price)
}
}
return map
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def ccbPriceMap){
def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
def price = ccbPriceMap.get(obj._id.toString())
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=20){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
}
}
return ret;
}
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.match_region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << CcbCommonUtils.getCcbCity(trade.city)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
ret << number
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
ret << ""
}
ret << trade.dist_id.toString()
ret << trade.master_name
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
ret << ""//期数
ret << trade.floor
ret << ""
ret << trade.height
// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.avg_price
ret << trade.total_price
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
def root = sourceMap.get(trade.source)
if(root){
if(CcbCommonUtils.rootMap.get(root)){
ret << CcbCommonUtils.rootMap.get(root)
}else{
ret << "99"
}
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.total_price
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){
def housetype = null
if(buildtype && ["0112","0110","0109"].contains(buildtype)){
housetype = "07"
}else if(buildtype && ["0107","0106","0105"].contains(buildtype)){
housetype = "16"
}else{
if(roomcnt && roomcnt==4){
if(livingroomcnt>1){
housetype = "14"
}else{
housetype = "11"
}
}else if(roomcnt && roomcnt==3){
if(livingroomcnt>1){
housetype = "13"
}else{
housetype = "10"
}
}else if(roomcnt && roomcnt==2){
if(livingroomcnt>1){
housetype = "12"
}else{
housetype = "09"
}
}else if(roomcnt && roomcnt==1){
housetype = "08"
}else{
housetype="99"
}
}
return housetype;
}
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all")
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
// if(price){
// if(dist.city=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else if(dist.city=="杭州"){
// startPrice = (int)(price - price*0.15)
// endPrice = (int)(price*0.15 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// }
// def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
// if(!trades && dist.address){
// def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
// if(flag){
// trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
//
// if(!trades){
// trades = CcbCommonUtils.findTradesDetailByAddressAlias(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
// }
// }
//
// }
def trades = CcbCommonUtils.findTradesDetailById(dist, startDate, endDate)
if(trades){
// def unitPrice= ppeDistsMap.get(dist._id.toString())
// if(unitPrice){
// for(def t : trades){
// t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
// }
// }
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return tradesList
}
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
// q.put("status", 0)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
def index = 0
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
rets << obj;
// def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
// if(price){
// def startPrice = 0
// def endPrice= 0
// if(obj.fjCity=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// if(obj.price>=startPrice && obj.price<=endPrice){
// rets << obj;
// }else{
// index++
// println index + " "+obj
// }
// }
}
}
return rets;
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", new BasicDBObject('$gte', weekend))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
def map = new ConcurrentHashMap()
for(def d : dbcursor){
def price = map.get(d.region+"##"+d.district_name)
if(!price && d.value_type==2){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
}
return map
}
def findHistoryDealIds(def city){
def sql = "SELECT gujia_bds_mdl_id from base.bigdata_deal where urbn_nm='${city}'"
def ids = []
try{
pgsqlInstance.eachRow(sql,{
ids << it.gujia_bds_mdl_id
});
}catch(Exception e){
}
return ids
}
def addDealPriceMatch(city){
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd")
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
final CountDownLatch latch = new CountDownLatch(trades.size())
def final AtomicInteger count_ = new AtomicInteger(0)
for(def i=0;i<trades.size();i++){
final def t = trades[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.addAndGet(1)
try{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
if(!rMap && t.address){
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
if(flag){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
}
}
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else {
if (rMap) {
println "区域不同"
}
}
}
println count_.get()+" "+t+" "+ rMap
}
}catch(Exception e){
e.printStackTrace()
}finally {
latch.countDown()
}
}
}
);
}
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
}
def delPgData(){
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
pgsqlInstance.execute("delete from base.bigdata_deal where p9_start_date='${month}'")
println "base.bigdata_deal 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_deal values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << "2023-09-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : newRows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,969 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.google.common.hash.*
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 出售房源数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataForsaleBatch extends BaseBatch{
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
runBatch(new FjwBigdataForsaleBatch(), args);
}
@Override
public void process(Object args) {
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19" //正式运行时curMonth要改
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = null;//findAllHouseIdsByCity()
for(def city : citys){
index++
houseIdsfilter = findAllHouseIdsByCity(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
}
}
def statCity(def citys){
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
def houseCitys = []
for(def d : datas){
houseCitys << d['C']
}
for(def city : citys){
if(!houseCitys.contains(city)){
def scity = CcbCommonUtils.specialCityMap.get(city)
if(!scity){
println city
}
}
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
saveToPgSql(allDatas)
}
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
// }
if(ret){
def areaData = distCodeMap.get(rMap.region)
if(areaData){
def regionCode = areaData.district_code
def cityCode = cityCodeDataMap.get(city)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
curhousefilter.put(m.houseId)
houseIdsfilter.put(m.houseId)
}else{
println city +" "+m.houseId+" 已存在"
}
}
}else{
println city +" "+rMap.region+" 没有找到对应的区域"
}
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
} catch (Exception e) {
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return retMap
}
def converData(def m){
def data = []
data << m.city_code
data << m.houseId
data << ""
data << m.city
data << m.region_code
data << m.region
data << m.district_id
data << m.district_name
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.address==null?"":m.address==null
data << m.build_year==null?"":m.build_year
data << m.proptype==null?"":m.proptype
data << m.propyears==null?"":m.propyears
data << m.avg_price==null?"":m.avg_price
data << m.total_price==null?"":m.total_price
data << ""
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.room_structure_code==null?"":m.room_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
data << m.house_type==null?"":m.house_type
data << m.room==null?"":m.room
data << m.hall==null?"":m.hall
data << ""
data << m.toilet==null?"":m.toilet
data << m.direction_code==null?"":m.direction_code
data << m.fitment_code==null?"":m.fitment_code
data << ""
data << ""
data << m.p_date==null?"":m.p_date
data << ""
data << ""
data << m.root
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.c_date==null?"":m.c_date
data << m.m_date==null?"":m.m_date
data << m.source
data << ""
return data
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
// def newRets = []
// def cityCode = cityCodeDataMap.get(city)
// def houseIds = getAllHouseIds(commId, cityCode)
// for(def r : rets1){
// if(!houseIds.contains(r.houseId)){
// newRets << r
// }else{
// println r.houseId +" 已存在"
// }
// }
// return newRets
// }
// return [];
return rets1;
}
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
//用以下方法会查出发布时间很久以前的
// QueryBuilder dateQueryBuilder = QueryBuilders.boolQuery()
// dateQueryBuilder.should(date_range)
// dateQueryBuilder.should(date_range_mdate)
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
// .must(area_range).must(dateQueryBuilder)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
def ids = new HashSet()
for(def h : rets1){
ids << h.houseId
}
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(ids && ids.contains(seawater.id)){
continue;
}
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
int retryCount = 0;
boolean success = false;
def rets = []
while (retryCount < MAX_TRY_COUNT && !success) {
try {
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
// 这里可以选择在重试之前等待一段时间
Thread.sleep(3000); // 例如等待1秒
}
}
if (!success) {
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
}
return rets;
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getPublicDate(def pDate){
if(pDate>=20181001 && pDate<20200201){
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
}
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
return pdate;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
private String csvHandlerStr(String str) {
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
String tempDescription=str;
//如果有逗号
if(str.contains(",")||str.contains(",")){
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
if(str.contains("\"")){
// tempDescription=str.replace("\"", "\"\"");
tempDescription=str.replace("\"", "");
}
//在将逗号转义
tempDescription="\""+tempDescription+"\"";
}
return tempDescription;
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
def marge(){
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
def ids = new HashSet();
def index = 0
file.eachLine{it->
index++
if(index>1){
def arrs = it.split("\\|@\\|")
def id = arrs[1]
if(!ids.contains(id)){
ids << id
}
}
}
println index
println ids.size()
}
def getAllHouseIds(def commid, def cityCode){
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'";
def houseIds = new HashSet()
sqlInstance.eachRow(sql,{it->
houseIds << it.FORSALEID
});
return houseIds;
}
def findAllHouseIdsByCity(cityid){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_id='${cityid}'",{it->
filter.put(it.gujia_sell_id)
});
// pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale_2021",{it->
// filter.put(it.gujia_sell_id)
// });
return filter;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_forsale values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40],
r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,235 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 人工指导价同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwGuiDingPriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwGuiDingPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
def priceIdsfilter = FjwPgsqlCommUtils.findGuiDingIdsByCity()
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
rows.addAll(rets)
println city + " " + rets.size()
}
}
println "【房价网内部】人工指导价总计:"+rows.size()
saveToPgSql(rows)
}
def findFangDatas(def districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println " 小区量:"+districts.size()
def result = Collections.synchronizedList(new ArrayList());
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findMatchThirdPrice(dist)
if(obj && obj.avg_price){
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def findPersonDatas(def districts, def dataMap){
def result = []
for(def i=0;i<districts.size();i++) {
def dist = districts[i];
def price = dataMap.get(dist._id.toString())
if (price) {
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
println dist.city + " " + dist.region + " " + dist.name + " " + price
}
}
return result;
}
def findMatchThirdPrice(dist){
def q = new BasicDBObject()
q.put("match_city", dist.city)
q.put("match_region", dist.region)
q.put("match_name", dist.name)
q.put("s_date", sDate)
q.put("source","搜房")
def obj = dataProcessColl.findOne(q)
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(priceIdsfilter.mightContain(ps._id.toString())){
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
continue
}
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code
ret << areaData.district
ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def createDatas(cityCodeDatas, distCodeMap, datas){
def rets = []
if(datas) {
for (def ps : datas) {
def ret = []
def areaData = distCodeMap.get(ps.region)
if (areaData) {
ret << ps.row_id
ret << ps.dist_id
ret << ps.name
ret << cityCodeDatas.get(ps.city)
ret << ps.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.avg_price
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], "all");
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.guided_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(300, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import groovy.sql.Sql
import java.nio.charset.StandardCharsets
/**
* 房价网内部使用
*/
class FjwPgsqlCommUtils {
public static def database = "fangjia"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
def sql = "select gujia_comm_id from base.bigdata_community where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def set = new HashSet()
sqlInstance.eachRow(sql,{r->
set << r.gujia_comm_id
});
return set
}
public static def findDistGovPrice(def city){
def ids = []
sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it->
ids << it.gujia_bds_id
});
return ids;
}
public static def deleteGuidedPrice(def dealdate){
def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'"
sqlInstance.executeUpdate(sql2)
}
public static def findGuiDingIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
sqlInstance.eachRow("SELECT gujia_fsid FROM base.guided_price",{it->
filter.put(it.gujia_fsid)
});
return filter
}
}
@@ -0,0 +1,553 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.concurrent.ConcurrentHashMap
/**
* 建行模型估值项目
* 第三方均价数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304//
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwThirdPartyPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
runDatas(citys)
//同步历史数据
// def stop = false
// def month = -1
// while(!stop){
// if(sDate>202404 && sDate<202407){
// def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
// println weekend.toString() +" "+sDate
// runDatas(citys)
// }
// month = month -1
// sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
// if(sDate == 201912){
// stop = true;
// }
// }
}
def runDatas(citys){
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
def gocIdsPrices = []
if(city == "深圳"){
gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city)
}
def statMap = getDistrictStatByCity(city, weekend)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all")
def thirdPriceMap = findThirdPartyPrice(city, statMap)
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices)
rows.addAll(rets)
println index+" "+city+" "+rets.size() +" "+rows.size()
}
saveToPgSql(rows)
}
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def rets = []
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
if(data){
def ret = []
ret << third.id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.get(d.city)
ret << d.city
ret << data.district_code
ret << data.district
ret << sDate
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
}else{
ret << ""
}
if(third.anjuke_esf_price){
ret << third.anjuke_esf_price
}else{
ret << ""
}
if(third.soufang_esf_price){
ret << third.soufang_esf_price
}else{
ret << ""
}
if(third.lianjia_esf_price){
ret << third.lianjia_esf_price
}else{
ret << ""
}
if(third.cityhouse_esf_price){
ret << third.cityhouse_esf_price
}else{
ret << ""
}
if(third.maitian_esf_price){
ret << third.maitian_esf_price
}else{
ret << ""
}
if(third.beike_esf_price){
ret << third.beike_esf_price
}else{
ret << ""
}
if(third.get("365house_esf_price")){
ret << third.get("365house_esf_price")
}else{
ret << ""
}
if(third.zhongyuan_esf_price){
ret << third.zhongyuan_esf_price
}else{
ret << ""
}
if(third.tuboshi_esf_price){
ret << third.tuboshi_esf_price
}else{
ret << ""
}
if(third.qfang_esf_price){
ret << third.qfang_esf_price
}else{
ret << ""
}
if(third.fangtan_esf_price){
ret << third.fangtan_esf_price
}else{
ret << ""
}
if(third.fangduoduo_esf_price){
ret << third.fangduoduo_esf_price
}else{
ret << ""
}
if(third.get("5i5j_esf_price")){
ret << third.get("5i5j_esf_price")
}else{
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){
def allRows = []
for(def d : dists){
if(gocPrices && gocPrices.contains(d._id.toString())){
continue;
}
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
if(data){
def dataMap = [:]
dataMap.COMMID = d._id.toString()
dataMap.COMMNAME = d.name
dataMap.CITYID = cityCodeDatas.get(d.city)
dataMap.CITYNAME = d.city
dataMap.DISTID = data.district_code
dataMap.DISTNAME = data.district
dataMap.YEAR_MONTH = sDate
def price_map = [:]
dataMap.price_map = price_map
if(third.zhugezhaofang_esf_price){
price_map.zhuge_avg_price = third.zhugezhaofang_esf_price
}
if(third.anjuke_esf_price){
price_map.anjuke_avg_price = third.anjuke_esf_price
}
if(third.soufang_esf_price){
price_map.fang_avg_price = third.soufang_esf_price
}
if(third.lianjia_esf_price){
price_map.lianjia_avg_price = third.lianjia_esf_price
}
if(third.cityhouse_esf_price){
price_map.cityhouse_avg_price = third.cityhouse_esf_price
}
if(third.maitian_esf_price){
price_map.maitian_avg_price = third.maitian_esf_price
}
if(third.beike_esf_price){
price_map.ke_avg_price = third.beike_esf_price
}
if(third.get("365house_esf_price")){
price_map.taofang_avg_price = third.get("365house_esf_price")
}
if(third.zhongyuan_esf_price){
price_map.zhongyuan_avg_price = third.zhongyuan_esf_price
}
if(third.tuboshi_esf_price){
price_map.tbs_avg_price = third.tuboshi_esf_price
}
if(third.qfang_esf_price){
price_map.qfang_avg_price = third.qfang_esf_price
}
if(third.fangtan_esf_price){
price_map.fangtan_avg_price = third.fangtan_esf_price
}
if(third.fangduoduo_esf_price){
price_map.fangduoduo_avg_price = third.fangduoduo_esf_price
}
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
}
}
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "房产超市"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
def rets = []
for (def d : dists) {
def third = thirdPriceMap.get(d.region + "##" + d.name)
if (third) {
def data = distCodeMap.get(d.region)
if (data) {
third.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << cityCodeDatas.get(d.city)
ret << data.district_code
ret << d._id.toString()
ret << d.name
def web = ""
if(k=="zhugezhaofang_esf_price"){
web = "诸葛找房"
}else if(k=="anjuke_esf_price"){
web = "安居客"
}else if(k=="soufang_esf_price"){
web = "房天下"
}else if(k=="lianjia_esf_price"){
web = "链家"
}else if(k=="cityhouse_esf_price"){
web = "禧泰数据"
}else if(k=="maitian_esf_price"){
web = "麦田"
}else if(k=="beike_esf_price"){
web = "贝壳"
}else if(k=="365house_esf_price"){
web = "淘房365"
}else if(k=="zhongyuan_esf_price"){
web = "中原"
}else if(k=="tuboshi_esf_price"){
web = "兔博士"
}else if(k=="qfang_esf_price"){
ret << "Q房"
}else if(k=="fangtan_esf_price"){
ret << "房探"
}else if(k=="fangduoduo_esf_price"){
ret << "房多多"
}else if(k=="5i5j_esf_price"){
ret << "我爱我家"
}else{
ret << ""
}
ret << sourceMap.get(web)
ret << web
ret << sDate
ret << v
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
dataMap.put("id", db._id.toString())
map.put(key, dataMap)
}
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
dataMap.put("qfang_esf_price", db.qfang_esf_price)
}
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
}
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
}
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
def startPrice = (int)(price - price*0.3)
def endPrice = (int)(price*0.3 + price)
value.each{pk,pv->
if(pk.contains("price")){
if(pv>=startPrice && pv<=endPrice){
priceMap.put(pk,pv)
}
}else{
priceMap.put(pk,pv)
}
}
if(priceMap.size()>1){
resultMap.put(key, priceMap)
}
}
}else{
resultMap.put(key, value)
}
}
}
return resultMap
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
return map
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
}
}
println "end: "+new Date()
}
}
@@ -13,7 +13,7 @@ class MsbPocTest {
task.run();
}
def run(){
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
@@ -24,7 +24,7 @@ class MsbPocTest {
// cityPrice()
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
addCCBPrice(inputfile)
}
def cityPrice(){
@@ -69,21 +69,21 @@ class MsbPocTest {
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['S'] && row['D']){
if(!row['Y'] && row['G']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['H']
def region = null //row['D']
def name = row['E']
def address = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['S'] = jsonObject.get("id")
row['T'] = jsonObject.get("city")
row['U'] = jsonObject.get("district")
row['V'] = jsonObject.get("name")
row['W'] = jsonObject.get("propertyType")
row['X'] = jsonObject.get("credit")
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("propertyType")
// row['X'] = jsonObject.get("credit")
}
}
})
@@ -111,23 +111,18 @@ class MsbPocTest {
def match(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
if(!row['Y']){
def city = row['C']
def region = row['D']
def region = null //row['B']
def address = row['G'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
if(jsonObject != null && jsonObject.credit>0) {
row['J'] = jsonObject.get("id")
row['K'] = jsonObject.get("city")
row['L'] = jsonObject.get("district")
row['M'] = jsonObject.get("name")
def type = jsonObject.get("propertyType")
if(type && type!='null'){
row['N'] = type
}
row['O'] = jsonObject.get("credit")
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(jsonObject != null) {
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
}
}
})
@@ -174,19 +169,22 @@ class MsbPocTest {
def addCCBPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['M']){
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
if(dist){
def d = CommonDatautils.findDistrictById(dist._id.toString())
if(!row['V']){
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
// if(dist){
if(row['Y'] instanceof String){
def d = CommonDatautils.findDistrictById(row['Y'])
if(d){
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
if(map){
row['M'] = map.avg_price
row['V'] = map.avg_price
// row['I'] = map.calc_type
}
}
}
// }
}
})
}
@@ -33,17 +33,31 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "C:\\Users\\gaojuan\\Desktop\\7Ô³ɽ»°¸Àýbadcase.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
exportPkPrice()
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
}
def addTradePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['Y']){
def dist = CommonDatautils.findDistrictById(d['Y'])
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
if (tradePrice) {
d['W'] = tradePrice.toInteger()
}
}
});
}
def addRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
@@ -155,13 +169,13 @@ class AddPriceTest {
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['M']){
def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['M'], cityId)
if(d['U']){
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(map){
d['AK'] = map.avg_price
d['AL'] = map.calc_type
d['AM'] = map.credit_value
d['F'] = map.avg_price
// d['P'] = map.calc_type
// d['AM'] = map.credit_value
}
}
})
@@ -169,7 +183,7 @@ class AddPriceTest {
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -190,11 +204,11 @@ class AddPriceTest {
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if (!d['X']) {
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['AN'] = thirdPrice
d['X'] = thirdPrice
}
}
})
@@ -206,8 +220,8 @@ class AddPriceTest {
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
q.put("s_date", new BasicDBObject('$gte', 202403))
// q.put("s_date", 202404)
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
for(def db : dbcursor){
@@ -216,10 +230,10 @@ class AddPriceTest {
break;
}
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
price = db.beike_esf_price
break
}
// if(!price && db.beike_esf_price && db.beike_esf_price>2000){
// price = db.beike_esf_price
// break
// }
}
return price
}
@@ -227,12 +241,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictById(d['E'])
if (!d['M']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['L'] = gxdPrice
d['M'] = gxdPrice
}
}
})
@@ -256,17 +270,28 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
// if(!d['L']){
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['AJ'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
d['E'] = dist.name
// d['D'] = dist.status
// d['E'] = dist.name_cat
// d['F'] = dist.estate_type2
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
// d['H'] = dist.master_district_name
// d['N'] = dist.region
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
if(dbo){
// d['K'] = dbo.avg_price
d['K'] = dbo.avg_price_2b
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['G'] = dbo.avg_price
// d['X'] = dbo.cal_type
// d['E'] = dbo.avg_price_2b
}
}
}
}
// }
// }
})
}
@@ -0,0 +1,172 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest {
String proxyHost = 'tunnel2.qg.net';
Integer proxyPort = 14186;
String proxyUsername = '113BBC06';
String proxyPassword = '3B2FBD05B0AF';
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
static main(args) {
QiliangTest task = new QiliangTest();
// task.run()
def data = [
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
}
def run(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\对公地址(问题案例)跑一下结果.xlsx", { row ->
if(row['B']){
def parmas = [
'address': row['B'],
'type': '1' //1:对公 0:对私
]
def str = getProxyResult(parmas);
println str
JSONObject result = JSONObject.fromObject(str)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['C'] = data.province
}
if(data.provinceCode && data.provinceCode!='null'){
row['D'] = data.provinceCode
}
if(data.city && data.city!='null'){
row['E'] = data.city
}
if(data.cityCode && data.cityCode!='null'){
row['F'] = data.cityCode
}
if(data.district && data.district!='null'){
row['G'] = data.district
}
if(data.districtCode && data.districtCode!='null'){
row['H'] = data.districtCode
}
if(data.town && data.town!='null'){
row['I'] = data.town
}
if(data.townCode && data.townCode!='null'){
row['J'] = data.townCode
}
if(data.village && data.village!='null'){
row['K'] = data.village
}
if(data.villageCode && data.villageCode!='null'){
row['L'] = data.villageCode
}
if(data.areaCode && data.areaCode!='null'){
row['M'] = data.areaCode
}
if(data.farmer == "0"){
row['N'] = "非农"
}else{
row['N'] = "农"
}
if(data.isGovt == "0"){
row['O'] = "否"
}else{
row['O'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['P'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['Q'] = data.govtAddr?data.govtAddr:''
}
}
}
}
})
}
String getProxyResult(data){
// 将 JSON 对象转换为字符串
String jsonData = new JsonOutput().toJson(data);
// 创建代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
// 创建 URL 对象
URL url = new URL(targetUrl);
// 创建连接对象
URLConnection connection = url.openConnection(proxy);
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST");
// 允许输出流
connection.setDoOutput(true);
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json");
// 发送请求参数
OutputStream outputStream = connection.getOutputStream();
outputStream.write(jsonData.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
// 读取响应
int responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
response.append(line);
}
reader.close();
return response.toString();
} else {
println "HTTP request failed with response code ${responseCode}";
}
return null;
}
}
@@ -200,4 +200,9 @@ public class SignatureUtils {
}
return null;
}
public static void main(String[] args){
def a = getAliAddressStrc("上海市长宁区新泾三村")
println a
}
}
@@ -18,77 +18,75 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
//def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
//ExcelUtils.multiThreadingModify(filePath, { row ->
//// if(!row['I']){
//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"ÈËÃñÕþ¸®")
//// if(rets){
//// def obj = findOneByName(rets, row['C']+"ÈËÃñÕþ¸®")
//// if(obj){
//// row['I'] = obj.lng
//// row['J'] = obj.lat
//// println row['C'] + " "+obj
//// }
//// }
//// }
//
//// if(row['D'] && !row['F']){
//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
//// def map = ["lon":row['D'], "lat":row['E']]
//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
//// row['F'] = township
//// }
//
//// if(row['F'] && !row['J']){
//// def q = new BasicDBObject()
//// q.put("city_name", row['B'])
//// q.put("area_name", row['C'])
//// def streetName = row['F']
//// q.put("street_name", streetName)
//// def count = COMMUNITY_CONN.count(q);
//// row['J'] = count
////// row['H'] = streetName
//// }
//// if(!row['H']){
//// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
//// if(gaodeMap && gaodeMap.township){
//// row['H'] = gaodeMap.township
//// }
//// }
//
//// if(row['J'] && row['J'].toString().toDouble()>0){
//// def up_query = new BasicDBObject()
//// up_query.put("city_name", row['B'])
//// up_query.put("area_name", row['C'])
//// up_query.put("street_name", row['F'])
//// def udata = new BasicDBObject()
//// udata.put("has_gov", "ÊÇ")
//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
//// }
//
//
//})
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
for(def d : objs){
if(!d.name_pinyin){
if(d.name){
println d._id
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
def udata = new BasicDBObject()
udata.put("name_pinyin", py1)
if(d.location){
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
udata.put("location", ["lat":lat, "lon":lon])
ExcelUtils.modify(filePath, { row ->
if(!row['I'] && row['I']=='宝应县'){
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
if(rets){
def obj = findOneByName(rets, row['C']+"人民政府")
if(obj){
row['I'] = obj.lng
row['J'] = obj.lat
println row['C'] + " "+obj
}
def q = new BasicDBObject()
q.put("_id", d._id)
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
}
}
}
if(row['D'] && !row['F']){
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
def map = ["lon":row['D'], "lat":row['E']]
def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
row['F'] = township
}
// if(row['F'] && !row['J']){
// def q = new BasicDBObject()
// q.put("city_name", row['B'])
// q.put("area_name", row['C'])
// def streetName = row['F']
// q.put("street_name", streetName)
// def count = COMMUNITY_CONN.count(q);
// row['J'] = count
//// row['H'] = streetName
// }
// if(!row['H']){
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
// if(gaodeMap && gaodeMap.township){
// row['H'] = gaodeMap.township
// }
// }
// if(row['J'] && row['J'].toString().toDouble()>0){
// def up_query = new BasicDBObject()
// up_query.put("city_name", row['B'])
// up_query.put("area_name", row['C'])
// up_query.put("street_name", row['F'])
// def udata = new BasicDBObject()
// udata.put("has_gov", "是")
// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
// }
})
//def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
//for(def d : objs){
// if(!d.name_pinyin){
// if(d.name){
// println d._id
// def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
// def udata = new BasicDBObject()
// udata.put("name_pinyin", py1)
// if(d.location){
// def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
// def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
// udata.put("location", ["lat":lat, "lon":lon])
// }
//
// def q = new BasicDBObject()
// q.put("_id", d._id)
// COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
// }
// }
//}
def getTownshipInfo(def glocation){
@@ -0,0 +1,181 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest1 {
def proxyHost = 'tunnel2.qg.net'
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
static main(args) {
def task = new QiliangTest()
// task.run()
def data = [
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
'type': '1'
]
def s = task.getProxyResult(data)
print(s)
}
def run(){//地址测试-2024.3.4-结果.xlsx
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
if(!row['N']){
def address = null
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
address = row['B']+row['C']
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
}
JSONObject getHttpResult(address){
JSONObject data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
JSONObject response = JSONObject.fromObject(res)
println response
return response
}catch(Exception e){
e.printStackTrace()
}
return null;
}
def getProxyResult(data){
// 将 JSON 对象转换为字符串
def jsonData = new JsonOutput().toJson(data)
// 创建代理对象
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
// 创建 URL 对象
def url = new URL(targetUrl)
// 创建连接对象
def connection = url.openConnection(proxy)
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST")
// 允许输出流
connection.setDoOutput(true)
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json")
// 发送请求参数
def outputStream = connection.getOutputStream()
outputStream.write(jsonData.getBytes("UTF-8"))
outputStream.flush()
outputStream.close()
// 读取响应
def responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def response = new StringBuilder()
String line
while ((line = reader.readLine()) != null) {
response.append(line)
}
reader.close()
return response.toString()
} else {
println "HTTP request failed with response code ${responseCode}"
}
return null
}
}
@@ -0,0 +1,265 @@
package com.fangjia.script.gj.xiamen
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticUtils89 {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
// connectionProperties.setProperty("user","developer");
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
connectionProperties.setProperty("user","elastic");
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
rs.close()
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
}
})
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
File diff suppressed because one or more lines are too long
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data
@@ -34,16 +35,18 @@ class PoiWash {
public static def regions = []
public static def nameKeys = null
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new PoiWash()
// task.updateId()
// task.run()
// task.synch()
// task.check()
task.deduplicate()
// task.deduplicate()
// task.updateEsMapPoi()
// task.checkSpecialSymbols()
// task.splitRoad()
task.splitRoad()
}
def updateId(){
@@ -452,9 +455,10 @@ class PoiWash {
def splitRoad(){
//"ÈýÃ÷ÊÐ",
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
def citys = CommonDatautils.findAllCity()
for(def city : citys) {
def datas = findPoiByQuery(city, ['city','address'])
def cityObj = fullCityMap.get(city)
def datas = findPoiByQuery(cityObj.city, ['city','address'])
splitRoadRun(datas)
println city +" Íê³É"
}
@@ -472,18 +476,15 @@ class PoiWash {
void run() {
try{
totalCount.addAndGet(1)
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
if(jsonObject && jsonObject.containsKey("result")){
def dataMap = jsonObject.get("result")
if(dataMap.get("street")){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("street", dataMap.get("street"))
if(dataMap.get("streetNumber")){
udata.put("street_number", dataMap.get("streetNumber"))
}
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
udata.put("m_date", new Date())
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
} catch (Exception e) {
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
@@ -26,8 +28,8 @@ class SeaweedDataSynch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
@@ -40,7 +42,7 @@ class SeaweedDataSynch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
@@ -61,11 +63,13 @@ class SeaweedDataSynch extends BaseBatch{
run()
// println "全国district, shop, office同步完成"
//民政部编码上传
//codeRun()
// codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
// addTest()
// updateTownship()
}
def addTest(){
@@ -125,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
def run(){
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
@@ -346,24 +350,44 @@ class SeaweedDataSynch extends BaseBatch{
def codeRun(){
// def q = new BasicDBObject()
// def dbcursor = mca_administrative_division_coll.find(q)
// def rets = []
// for(def it : dbcursor){
// def udata = new BasicDBObject()
// udata.put("_id", it._id)
// udata.put("province", it.province)
// udata.put("city", it.city)
// udata.put("district", it.district)
// udata.put("mca_province_code", it.province_code)
// udata.put("mca_city_code", it.city_code)
// udata.put("mca_district_code", it.district_code)
// udata.put("level", it.level)
// udata.put("c_date", it.c_date)
// rets << udata
//// administrative_division_100_CONN.save(udata)
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
// }
// administrative_division_100_CONN.insert(rets)
//
def q = new BasicDBObject()
def dbcursor = mca_administrative_division_coll.find(q)
def rets = []
for(def it : dbcursor){
def udata = new BasicDBObject()
udata.put("_id", it._id)
udata.put("province", it.province)
udata.put("city", it.city)
udata.put("district", it.district)
udata.put("mca_province_code", it.province_code)
udata.put("mca_city_code", it.city_code)
udata.put("mca_district_code", it.district_code)
udata.put("level", it.level)
udata.put("c_date", it.c_date)
rets << udata
// administrative_division_100_CONN.save(udata)
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
q.put("province","澳门特别行政区")
def dbcursor = administrative_division_2022_CONN.find(q)
for(def d : dbcursor){
administrative_division_100_CONN.save(d)
}
administrative_division_100_CONN.insert(rets)
}
def updateTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
if(row['Q'] && row['Q']=='是'){
def udata = new BasicDBObject()
udata.put("township", row['J'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
}
})
}
}
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong
class SuggestionDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
@@ -1,5 +1,6 @@
package com.fangjia.script.hubingliang
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import groovy.sql.Sql
@@ -23,7 +24,7 @@ import com.mongodb.DBObject
@BatchOwner(Watcher.Hubingliang)
class ApiInvokeStatistics extends BaseBatch{
final static def mongoDomain = "mongo.db.fangjia.com"
public static final DBCollection OPEN_URI_LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com", 27020)
public static final DBCollection OPEN_URI_LOG = com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI,"business", "open_uri_log")
public static final DBCollection OPEN_URI_LOG_COUNT = MongoUtils.getCollectionByName("business", "api_invoke_statistical", "mongo.db.fangjia.com", 27017)
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
static main(args) {
@@ -98,6 +98,23 @@ class InsidePolygon {
}
return list;
}
public static List<List<Point>> createPoint4(def pointList){
List<List<Point>> alllist = new ArrayList<>();
for(def pointArr : pointList){
List<Point> list = new ArrayList<>();
for(def arr : pointArr){
Point point = new Point();
double x = arr[0];
double y = arr[1];
point.setLon(x);
point.setLat(y);
list.add(point);
}
alllist << list
}
return alllist
}
/**
* 判定一个经纬度点与区域的关系
@@ -284,7 +301,7 @@ class InsidePolygon {
if(flag){
def arrs = key.split("##")
name = arrs[1];
break;
// break;
}
}
if(flag){
@@ -0,0 +1,50 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\1.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count = count + 1
if (count % 10 == 0) {
println("已处理:" + count)
}
sleep(80)
def address = items['A']
def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) {
def province = map.get("prov")
items['E'] = province
}
if ("city" in map) {
def city = map.get("city")
items['G'] = city
}
if ("district" in map) {
def district = map.get("district")
items['I'] = district
}
if ("town" in map) {
def town = map.get("town")
items['K'] = town
}
// if ("community" in map) {
// def community = map.get("community")
// items['S'] = community
// }
if ("devzone" in map) {
def devzone = map.get("devzone")
items['Q'] = devzone
}
})
@@ -0,0 +1,85 @@
package com.fangjia.script.majintao.JSBankTest;
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils;
import net.sf.json.JSONObject;
import java.util.Base64;
import net.sf.json.JSONArray;
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count += 1
if (count % 10 == 0) {
println("已处理:" + count)
}
def zx = ["北京", "天津", "上海", "重庆"]
// String province = "天津";
String province = items['E'];
if (zx.contains(province)) {
province = province + "市";
}
// String name = "天津经济技术开发区";
String name = items['Q'];
String elasticUsername = "elastic";
String elasticPassword = "elasticsearch";
String seawaterUrl = "http://192.168.0.89:9200/development_zone/_search";
String basicAuth = "Basic " + Base64.getEncoder().encodeToString((elasticUsername + ":" + elasticPassword).getBytes("UTF-8"));
String body = """
{
"query": {
"bool": {
"must": [
{
"match": {
"name": {
"query": "${name}",
"operator": "and"
}
}
},
{
"term": {"province": {"value": "${province}"}}
}
]
}
}
}
""";
HttpRequest request = new HttpRequest();
JSONObject jsonBody = JSONObject.fromObject(body);
JSONObject header = new JSONObject();
header.put("Authorization", basicAuth);
String res = request.postWithBody(seawaterUrl, jsonBody, header, "UTF-8");
JSONObject response = JSONObject.fromObject(res);
JSONObject obj = null;
if (response != null && response.containsKey("hits")) {
JSONObject hits = response.getJSONObject("hits");
if (hits != null && hits.size() > 0) {
JSONArray hitsArray = hits.getJSONArray("hits");
// 省略了之前的部分...
if (hitsArray != null && hitsArray.size() > 0) {
for (int i = 0; i < hitsArray.size(); i++) {
JSONObject hit = hitsArray.getJSONObject(i);
JSONObject source = hit.getJSONObject("_source");
String devName = source.optString("name"); // Assuming "name" is a String
String level = source.optString("level"); // Assuming "level" is a String
if (devName.equals(name)) {
items['O'] = "是";
items['P'] = level;
println("省份:" + province + " 名称:" + name + " 开发区名:" + devName + " 开发区级别:" + level);
break
}
}
}
}
}
});
@@ -0,0 +1,85 @@
package com.fangjia.script.majintao.JSBankTest
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import org.apache.tools.ant.types.resources.selectors.None
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count += 1
if (count % 10 == 0) {
println("已处理:" + count)
}
String address = items['B']
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("showFullValue", "2")
map.put("showTestValue", "2")
map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if (result != null) {
if ("addrVerify") {
items['C'] = result.get("addrVerify")
}
if ("province" in result) {
items['D'] = result.get("province")
}
if ("city" in result) {
items['F'] = result.get("city")
}
if ("district" in result) {
items['H'] = result.get("district")
}
if ("town" in result) {
items['J'] = result.get("town")
}
if ("committee" in result) {
items['L'] = result.get("committee")
}
if ("typeCode" in result) {
items['M'] = result.get("typeCode")
}
if ("countyGov") {
items['N'] = result.get("countyGov")
}
def devDistrict = ""
if ("devDistrict" in result) {
devDistrict = result.get("devDistrict")
}
def dev = ""
if ("dev" in result) {
dev = result.get("dev")
}
// 开发区级别暂时没有
// 暂时使用
def defGrade = ""
if (!dev.equals("") && devDistrict.equals("否")) {
defGrade = "非国际级或省级"
devDistrict = "是"
}
if (!devDistrict.equals("")) {
items['O'] = devDistrict
}
items["P"] = defGrade
}
})
@@ -5,7 +5,11 @@ import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
<<<<<<< HEAD
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx";
=======
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
@@ -14,13 +18,43 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
if (count % 10 == 0) {
println("已处理:" + count)
}
<<<<<<< HEAD
sleep(80)
=======
sleep(50)
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
def address = items['A']
def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) {
def province = map.get("prov")
<<<<<<< HEAD
items['C'] = province
}
if ("city" in map) {
def city = map.get("city")
items['G'] = city
}
if ("district" in map) {
def district = map.get("district")
items['K'] = district
}
if ("town" in map) {
def town = map.get("town")
items['O'] = town
}
if ("community" in map) {
def community = map.get("community")
items['S'] = community
}
if ("devzone" in map) {
def devzone = map.get("devzone")
items['AE'] = devzone
}
=======
items['D'] = province
}
if ("city" in map) {
@@ -66,6 +100,7 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
// items['K'] = roadno
// }
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
})
@@ -4,10 +4,17 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
<<<<<<< HEAD
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
=======
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
count += 1
if (count % 10 == 0) {
println("已处理:" + count)
@@ -15,6 +22,112 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
String address = items['A']
<<<<<<< HEAD
// if (province == null) {
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("showFullValue", "2")
map.put("showTestValue", "2")
map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if (result != null) {
if ("province" in result) {
items['B'] = result.get("province")
}
if ("provinceCode" in result) {
items['E'] = result.get("provinceCode")
}
if ("city" in result) {
items['F'] = result.get("city")
}
if ("cityCode" in result) {
items['I'] = result.get("cityCode")
}
if ("district" in result) {
items['J'] = result.get("district")
}
if ("districtCode" in result) {
items['M'] = result.get("districtCode")
}
if ("town" in result) {
items['N'] = result.get("town")
}
if ("townCode" in result) {
items['Q'] = result.get("townCode")
}
if ("committee" in result) {
items['R'] = result.get("committee")
}
if ("committeeCode" in result) {
items['U'] = result.get("committeeCode")
}
if ("typeCode" in result) {
items['V'] = result.get("typeCode")
}
if ("typeName") {
items['W'] = result.get("typeName")
}
if ("verifyFlag") {
items['X'] = result.get("verifyFlag")
}
if ("addrVerify") {
items['Y'] = result.get("addrVerify")
}
if ("countyLevelCity") {
items['Z'] = result.get("countyLevelCity")
}
if ("countyGov") {
items['AA'] = result.get("countyGov")
}
if ("communitySource") {
items['AB'] = result.get("communitySource")
}
if ("administrativeTowns" in result) {
items['AD'] = result.get("administrativeTowns")
}
if ("devDistrict" in result) {
items['AF'] = result.get("devDistrict")
}
if ("devDistrictName" in result) {
items['AG'] = result.get("devDistrictName")
}
if ("splitCommittee" in result) {
items['AH'] = result.get("splitCommittee")
}
if ("splitStreet" in result) {
items['AI'] = result.get("splitStreet")
}
if ("poiName" in result) {
items['AJ'] = result.get("poiName")
}
if ("splitStreetNum" in result) {
items['AK'] = result.get("splitStreetNum")
}
if ("dev" in result) {
items['AL'] = result.get("dev")
}
}
// }
=======
String province = items['B']
if (province == null) {
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
@@ -120,5 +233,6 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
}
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
})
@@ -24,8 +24,13 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
def map = [:]
map.put("address", address)
<<<<<<< HEAD
map.put("complete", "2")
map.put("addressAssessment", "2")
=======
map.put("complete", "true")
map.put("addressAssessment", "true")
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
@@ -0,0 +1,15 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
String url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("showFullValue", "2")
map.put("showTestValue", "2")
map.put("type", "2")
@@ -1,5 +1,6 @@
package com.fangjia.script.openApi
import com.fangjia.core.Constants
import groovy.sql.Sql;
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -24,7 +25,7 @@ import com.fangjia.script.Watcher
class OpenApiLogAnalyBatch extends BaseBatch{
static DBCollection OPEN_URI_LOG_COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com",27020);
static DBCollection OPEN_URI_LOG_COLL = com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI,"business", "open_uri_log")
static DBCollection OPEN_URI_ANALY_COLL = MongoUtils.getCollectionByName("business", "open_uri_analy", "mongo.db.fangjia.com");
final Sql USER_CONN = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?useUnicode=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver")
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
* 8:地址及地址别名查找
* 9:地址及地址别名(不加区域限制)查找
* 10:拼音匹配
*11:open接口匹配
* 11:open接口匹配
* 14:拼音匹配,不加地址限制
*/
@BatchOwner(Watcher.GaoJuan)
@@ -61,10 +61,10 @@ class DataStandardBatch extends BaseBatch{
private static def districtMap = [:];
private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth()
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
//查找出所有未进行匹配的数据
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","福州")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
@@ -23,8 +23,8 @@ class DataStat {
// for(def city :citys){
// println city
// }
// task.run();
task.stat()
task.run();
// task.stat()
// def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"]
// def rets = []
@@ -76,7 +76,7 @@ class DataStat {
}
def stat(){
def dates = [202301,202302,202303,202304,202305,202306,202307,202308,202309,202310,202311,202401]
def dates = [202303,202304,202305,202306,202307,202308,202309,202310,202311,202401,202402]
def sources = ["时间","我爱我家","兔博士","吉屋","贝壳找房","乐有家","365淘房","诸葛找房","链家","Q房","麦田","中原地产","安居客","搜房","C21","城市房产","幸福里"]
// crawlCount(dates,sources);
matchCount(dates,sources);
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["搜房":"搜房"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
for(def source :sources){
def count1 = findTotalCount(source, 202307);
def count2 = findTotalCount(source, 202401);
def count1 = findTotalCount(source, 202401);
def count2 = findTotalCount(source, 202402);
println source +" "+count1 +" "+count2
}
}
@@ -32,7 +32,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -0,0 +1,69 @@
package com.fangjia.script.util
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
public class DESUtil {
private static final String ALGORITHM = "DES";
// 生成密钥
public static SecretKey generateKey() throws Exception {
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
keyGenerator.init(56); // DES密钥长度为56位
return keyGenerator.generateKey();
}
// 将密钥转换为字符串
public static String keyToString(SecretKey secretKey) {
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
}
// 将字符串转换为密钥
public static SecretKey stringToKey(String keyStr) {
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
}
// DES加密
public static String encrypt(String data, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
byte[] encryptedData = cipher.doFinal(data.getBytes());
return Base64.getEncoder().encodeToString(encryptedData);
}
// DES解密
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.DECRYPT_MODE, secretKey);
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
byte[] decryptedData = cipher.doFinal(decodedData);
return new String(decryptedData);
}
public static void main(String[] args) {
try {
// 生成密钥
// SecretKey secretKey = generateKey();
// String keyStr = keyToString(secretKey);
// System.out.println("密钥: " + keyStr);
//
//// // 加密数据
// String data = "高娟";
// String encryptedData = encrypt(data, secretKey);
// System.out.println("加密后的数据: " + encryptedData);
// 解密数据
SecretKey newSecretKey = stringToKey("tnb12345");
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
System.out.println("解密后的数据: " + decryptedData);
} catch (Exception e) {
e.printStackTrace();
}
}
}
@@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.*
public class DbUtils{
public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com";
public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
//public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
public static final String COMMON_DB = "fangjia";
public static final int COMMON_PORT = 27017;
public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
//public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
@@ -186,6 +186,26 @@ public class DbUtils{
return query
}
public static BasicDBObject buildSeaweedQuery2(def city = null, def region = null, def name = null, def cat = "district", def status = 0){
BasicDBObject query = new BasicDBObject()
if(city){
query.append("city", city)
}
if(region){
query.append("region", region)
}
if(name){
query.append("name", name)
}
if(cat != null){
query.append("cat", cat)
}
if(status != null){
query.append("status", status)
}
return query
}
/**
* °ÑÐ¡ÇøÐ޸ijÉɾ³ý״̬
* @param id
@@ -76,7 +76,6 @@ class ElasticUtils {
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@@ -101,6 +100,30 @@ class ElasticUtils {
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 3
def nThread = Runtime.getRuntime().availableProcessors() * 4
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
@@ -454,8 +454,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = 10
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(3)
final AtomicInteger counter = new AtomicInteger(0)
final double total = body.size()
@@ -71,6 +71,10 @@ class MongoUtils {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
}
if ((domainUrl == "192.168.0.206") && (port == 27020)) {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name)
}
def key = db + name + domainUrl + port
if (collectionMap.containsKey(key)) {
return collectionMap[key]
@@ -156,6 +156,22 @@ public class ProvinceData {
return PROVINCE_MAP.get(province);
}
public static String getSimpleProvince(String province){
def prov = null
def newDatas = data + GOVERNMENT_CITY_LIST
for(def key : newDatas.keySet()){
def list = newDatas.get(key)
list << key
if(list.contains(province)){
prov = key
break;
}
}
return prov;
}
/**
* 获取省份下的所有城市
* @param province

Some files were not shown because too many files have changed in this diff Show More