This commit is contained in:
2023-03-15 14:30:20 +08:00
14 changed files with 1739 additions and 858 deletions
@@ -46,7 +46,7 @@
<div class="banner mobile02" id="company" style="background:url(${staticUrl}/images/fangjia/v5/about_banner_m.jpg) no-repeat center center;background-size:100%;">
<div>
<p><label>时空价值数字化技术的引领者</label></p>
<p><small style="font-size:24px;line-height:30px;">致力于建立最全面的时空价值图谱,发现人、物和场的深度关联,为政府、<br/>金融、零售
<p><small style="font-size:14px;line-height:30px;">致力于建立最全面的时空价值图谱,发现人、物和场的深度关联,为政府、<br/>金融、零售
等提供风控、营销、规划等场景的产品和解决方案</small></p>
</div>
</div>
@@ -26,6 +26,15 @@
background-size: 100% 100%;
margin-top: -2px;
}
.img_m2{
image-rendering: -moz-crisp-edges;
image-rendering: -o-crisp-edges;
image-rendering: -webkit-optimize-contrast;
image-rendering: crisp-edges;
-ms-interpolation-mode: nearest-neighbor;
}
</style>
<script>
if((navigator.userAgent.match(/(iPad|iPhone|iPod|Android|ios)/i))){
@@ -63,7 +72,7 @@
/*box-shadow: 0 0 10px rgba(0,0,0,0.3), 0 0 10px rgba(0,0,0,0.3) inset;*/
}
.column > div p {
.radius .column > div p {
width: 500px;
}
ul { list-style: none; margin: 0; padding: 0; }
@@ -194,7 +203,7 @@
<#-- </div>-->
<#--</div>-->
<div class="core-main">
<div class="core-main mobile01">
<div class="columnBox firstOne firstOneM">
<div class="column">
<div class="column_center2 mobile01" style="padding-left: 36px;padding-top: 40px;">
@@ -205,27 +214,39 @@
</div>
<center>
<div class="columnBox firstOneM" style="padding: 20px 0 20px 0;">
<div class="columnBox firstOneM mobile01" style="padding: 20px 0 20px 0;">
<div class="column">
<img src="${staticUrl}/images/fangjia/v5/new-index2.png" style="width: 94%;"><br>
</div>
</div>
</center>
</div>
<div class="columnBox firstOne firstOneM" style="padding: 120px 0 20px 0;">
<div class="columnBox mobile02" style="margin-top: 50px;">
<p>能力全景图</p>
</div>
<div class="columnBox bg mobile02">
<div class="column">
<div class="column_center2 mobile01" style="padding-left: 36px;">
<div class="column_left">
<img src="${staticUrl}/images/fangjia/v5/new-index2.png">
</div>
</div>
</div>
<div class="columnBox firstOne firstOneM mobile01" style="padding: 120px 0 20px 0;">
<div class="column">
<div class="column_center2 " style="padding-left: 36px;">
<p style="width: 100%;font-size: 24px;font-weight: bold;color: #666666;">解决方案</p>
</div>
</div>
</div>
<center>
<div class="radius" style="height: 280px;">
<div class="radius mobile01" style="height: 280px;">
<div class="columnBox firstOneM" style="padding: 20px 0 80px 0;">
<div class="column" style="padding-left: 25px;">
<div style="display: flex;">
@@ -280,23 +301,78 @@
</div>
</center>
<div class="columnBox firstOne firstOneM" style="margin: 50px 0 20px 0;">
<div class="columnBox mobile02">
<p>解决方案</p>
</div>
<div class="columnBox mobile02">
<div class="column">
<div class="column_center2 mobile01" style="padding-left: 36px;">
<div class="column_left" style="margin-top:0;">
<img src="${staticUrl}/images/fangjia/v5/appraisal_img05.png">
</div>
<div class="column_right">
<p><label>押品全生命周期价值管理</label></p>
<p><small>围绕押品信贷业务全生命周期,从营销展业,贷前、贷中、贷后、以及业务运营形成一体式的风险防控,通过建立房贷业务地图平台,实现全方位监测。</small></p>
</div>
</div>
</div>
<div class="columnBox mobile02">
<div class="column">
<div class="column_left" style="margin-top:0;">
<img src="${staticUrl}/images/fangjia/v5/consulting_img03.jpg">
</div>
<div class="column_right">
<p><label>地址治理</label></p>
<p><small>地址数据治理除了以地址标准化为核心的一系列智能化服务,更具有为各行各业建立以地址(位置)信息为基础的数据库与价值挖掘体系的能力。夯实数据资产数字化建设,全面赋能业务场景。</small></p>
</div>
</div>
</div>
<div class="columnBox mobile02">
<div class="column">
<div class="column_left" style="margin-top:0;">
<img src="${staticUrl}/images/fangjia/v5/outlets.png">
</div>
<div class="column_right">
<p><label>网点布局规划</label></p>
<p><small>建设智慧型社区银行、构建新一轮的智慧网点布局战略和实施规划,地址价值挖掘全面提升网点布局规划的科学性、
专业性及合理性,强化社区融合、占据市场主动、培育优质客户。</small></p>
</div>
</div>
</div>
<div class="columnBox mobile02">
<div class="column">
<div class="column_left" style="margin-top:0;">
<img src="${staticUrl}/images/fangjia/v5/bank_img01.jpg">
</div>
<div class="column_right">
<p><label>基于地理位置的财力评级</label></p>
<p><small>地理位置与经济实力息息相关<br/>
居住品质是甄别财富客户的有效指征<br/>
地址价值挖掘,用AI算法深度解析地址背后的信息,助力企业找到“对的人”</small></p>
</div>
</div>
</div>
<div class="columnBox firstOne firstOneM mobile01" style="margin: 50px 0 20px 0;">
<div class="column">
<div class="column_center2" style="padding-left: 36px;">
<p style="width: 100%;font-size: 24px;font-weight: bold;color: #666666;">产品推荐</p>
</div>
</div>
</div>
<center>
<div class="radius" style="height: 335px;">
<div class="columnBox firstOne firstOneM" style="margin-top: 20px;">
<div class="radius mobile01" style="height: 335px;">
<div class="columnBox firstOne firstOneM" style="margin-top: 60px;">
<div class="column">
<div class="column_left mobile01" style="margin-left:40px;">
<img src="${staticUrl}/images/fangjia/v5/ds2.png" width="440" height="auto">
</div>
<div class="column_left mobile02">
<img src="${staticUrl}/images/fangjia/v5/ds2.png" width="440" height="auto">
<img src="${staticUrl}/images/fangjia/v5/bank_img01.jpg" width="440" height="auto">
</div>
<div class="column_left">
<div>
@@ -309,22 +385,59 @@
</div>
</center>
<div class="columnBox firstOne firstOneM" style="padding: 120px 0 40px 0;">
<div class="columnBox mobile02">
<p>产品推荐</p>
</div>
<div class="columnBox mobile02">
<div class="column">
<div class="column_left">
<img src="${staticUrl}/images/fangjia/v5/ds2.png" width="440" height="auto">
</div>
<div class="column_left">
<center>
<div>
<h1 style="font-size: 20px;">地数— 地址价值挖掘平台</h1>
<p style="padding-top: 20px;"><span style=" font-size: 16px; line-height: 30px;">&nbsp;&nbsp;&nbsp;&nbsp;“地数”是估家科技在多年积累的房地数据和行业经验基础上,自主研发的地址大数据价值挖掘产品。这一平台的核心是强大的地址数据处理技术和价值计算模型,实现对各类机构、企业相关的位置数据进行自动化的治理,并结合不同场景进一步挖掘其业务价值。</span></p>
</div>
</center>
</div>
</div>
</div>
<div class="columnBox firstOne firstOneM mobile01" style="padding: 120px 0 40px 0;">
<div class="column">
<div class="column_center2 mobile01" style="padding-left: 36px;">
<p style="width: 100%;font-size: 24px;font-weight: bold;color: #666666;">典型客户</p>
</div>
</div>
</div>
<center>
<div class="columnBox firstOne firstOneM" style="padding: 40px 0 80px 0;">
<div class="columnBox firstOne firstOneM mobile01" style="padding: 40px 0 80px 0;">
<div class="column">
<img src="${staticUrl}/images/fangjia/v5/company.png" style="width: 96%;height: 400px;"><br>
</div>
</div>
</center>
<div class="columnBox mobile02">
<p>典型客户</p>
</div>
<div class="columnBox mobile02">
<div class="column">
<div class="column_left" style="margin-bottom:1rem;margin-top: 0rem">
<img src="${staticUrl}/images/fangjia/v5/partner_m1.png" class="img_m2">
</div>
</div>
</div>
<#--<#if adsList?? && adsList?size gt 0>-->
<#-- <#list adsList as ad>-->
<#-- <img src="${ad.image!}" class="house_img" onclick=clickUrl("${ad.id}","${ad.url!}")>-->
@@ -76,7 +76,7 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
saveToPgSql(rows)
// saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
@@ -10,6 +10,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.google.common.hash.*
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
@@ -20,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -56,7 +58,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
@@ -76,16 +78,20 @@ class BigdataForsaleBatch extends BaseBatch{
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
citys = ["迪庆州"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def houseIdsfilter = findAllHouseIdsByCity()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319, houseIdsfilter);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
@@ -108,14 +114,14 @@ class BigdataForsaleBatch extends BaseBatch{
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate)
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
allDatas = repeatHouseId(allDatas)
println city + "(新)房源总量:"+allDatas.size()
// // allDatas = repeatHouseId(allDatas)
// println city + "(新)房源总量:"+allDatas.size()
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
@@ -136,17 +142,19 @@ class BigdataForsaleBatch extends BaseBatch{
saveToPgSql(allDatas)
}
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try {
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
@@ -168,23 +176,27 @@ class BigdataForsaleBatch extends BaseBatch{
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
if(!houseIdsfilter.mightContain(m.houseId)){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
}else{
println city +" "+m.houseId+" 已存在"
}
}
}else{
println city +" "+rMap.region+" 没有找到对应的区域"
@@ -198,8 +210,8 @@ class BigdataForsaleBatch extends BaseBatch{
} finally {
latch.countDown()
}
};
})
// };
// })
}
latch.await()
return retMap
@@ -482,7 +494,7 @@ class BigdataForsaleBatch extends BaseBatch{
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
@@ -1090,6 +1102,20 @@ class BigdataForsaleBatch extends BaseBatch{
return houseIds;
}
def findAllHouseIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h where urbn_id='533400'",{it->
filter.put(it.gujia_sell_id)
});
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021 where urbn_id='533400'",{it->
filter.put(it.gujia_sell_id)
});
return filter;
}
def isExistFromHouse(def houseId){
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
@@ -1109,7 +1135,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -45,7 +45,7 @@ class CcbCommonUtils {
public static def sqlInstance = null
public static def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
public static def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -127,12 +127,12 @@ class CcbCommonUtils {
def q = new BasicDBObject()
q.put("city", city)
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def dbcursor = GUIDING_PRICE_COLL.find(q).sort(new BasicDBObject("guiding_date",-1))
def rets = []
def sets = new HashSet()
for(def d : dbcursor){
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
def key = d.city +"##"+d.region+"##"+d.name
if(!sets.contains(key)){
rets << d
sets << key
@@ -454,6 +454,41 @@ class CcbCommonUtils {
return rets;
}
public static def findAllDistrictByCityAndRegion(def city, def region, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", region)
// q.put("name","鸣翠蓝湾")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
println "原始:"+dbsour.count()
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
rets << d
}
}
println "小区量:"+rets.size()
return rets;
}
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
@@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
// "postgres", "postgres", "org.postgresql.Driver")
//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -1172,18 +1172,39 @@ def findThirdCommCount(cityid, sqlInstance){
// return map
//}
//
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['G'] = dist.city
// d['H'] = dist.region
// d['I'] = dist.name
// }
//})
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['G'] = dist.city
d['H'] = dist.region
d['I'] = dist.name
}
})
//def findAvgPrice(def commid, def pgsqlInstance, def cityid){
// def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
// def map = [:]
// pgsqlInstance.eachRow(sql, { r ->
// map.avg_price = r.gujia_bds_clc_avprc
// map.type = r.gujia_bds_clmtd
// map.month = r.gujiabds_clc_yrmo_val
// if(r.gujia_bds_clmtd == 1){
// map.calc_type = "人工估值"
// }else if(r.gujia_bds_clmtd == 2){
// map.calc_type = "样本计算"
// }else if(r.gujia_bds_clmtd == 3){
// map.calc_type = "神经网络"
// }
// })
// return map
//}
//def file = "C:\\Users\\gaojuan\\Desktop\\国信达小区.xlsx"
//def cityCodeMap = CcbCommonUtils.findCityCodeValue()
//ExcelUtils.multiThreadingModify(file, { d ->
@@ -178,9 +178,9 @@ class ExportPgsqlEvaluationResult {
});
}
def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false)
}
@@ -207,9 +207,9 @@ class ExportPgsqlEvaluationResult {
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false)
@@ -369,27 +369,28 @@ class ExportPgsqlEvaluationResult {
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}";
def allRows = []
pgsqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_fsid
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_case_id
data << r.gujia_bds_crterm
data << r.data_crt_tm
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
// allRows.clear()
// }
});
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
// for(def city : citys){
// def cityCode = cityCodeMap.get(city)
// def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}";
// def allRows = []
// pgsqlInstance.eachRow(sql,{r->
// def data = []
// data << r.gujia_fsid
// data << r.urbn_id
// data << r.gujia_comm_id
// data << r.gujia_case_id
// data << r.gujia_bds_crterm
// data << r.data_crt_tm
// allRows << data
//// if(allRows.size()%200000==0){
//// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
//// allRows.clear()
//// }
// });
//
// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
// }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
}
@@ -20,6 +20,10 @@ class ExportDistrictRongHe extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new ExportDistrictRongHe(), args);
}
@@ -29,7 +33,6 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
@@ -37,8 +40,10 @@ class ExportDistrictRongHe extends BaseBatch{
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()//
@@ -63,26 +68,30 @@ class ExportDistrictRongHe extends BaseBatch{
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
exportThirdPartyPrice2(thirdPartyMonth, new_dists)
// exportThirdPartyPrice2(thirdPartyMonth, new_dists)
exportPgThirdPartyPrice2(thirdPartyMonth, new_dists)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
exportCommunityBasePrice(lastMonth)
exportCommunityPdescribeGj(lastMonth)
exportGuideStatistics(lastMonth)
exportBigdataSupplierStatistics(lastMonth)
exportBigdataModelingBase(lastMonth)
// exportCommunityBasePrice(lastMonth)
// exportCommunityPdescribeGj(lastMonth)
// exportGuideStatistics(lastMonth)
// exportBigdataSupplierStatistics(lastMonth)
// exportBigdataModelingBase(lastMonth)
exportPgCommunityBasePrice(lastMonth)
exportPgCommunityPdescribeGj(lastMonth)
exportPgGuideStatistics(lastMonth)
exportPgBigdataSupplierStatistics(lastMonth)
exportPgBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出
exportPgBigdataForsale("2023-01-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
}
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
}
//
@@ -579,6 +588,46 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def exportPgBigdataDealFJW(curDate, dealdate, new_dists){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("10")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealdate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>='${dealdate}'";
allRows.addAll(dealData(sql))
}
}
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def createDeal(sql){
def allRows = []
sqlInstance.eachRow(sql,{r->
@@ -658,6 +707,87 @@ class ExportDistrictRongHe extends BaseBatch{
return allRows;
}
def dealData(def sql){
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_bds_mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bds_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_bds_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujiabdstot_floor_num
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.guiabconstructionarea
data << r.gujiabdsset_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_bds_room_num
data << r.gujia_bds_dept_num
data << r.gujia_bds_ktch_num
data << r.gujia_bds_hlt_btw_num
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_bds_trdtckt_prc
data << r.gujia_bds_mdl_tot_prc
data << r.gujia_bds_dcl_tot_prc
data << r.gujiabds_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_bds_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiabdslndfgrngvarea
data << r.gujiabdslutbl_gv_area
data << r.gujiabdssuntblgv_area
data << r.gujiabdsbayindowgarea
data << r.gujiabdstowerfrgvarea
data << r.gujia_bds_grg_gv_area
data << r.gujiabdsundgdrmgvarea
data << r.gujiabds_othr_gv_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
// data << r.gujia_bds_mkdam
data << r.gujia_bds_data_tm_val
allRows << data
});
return allRows
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = []
@@ -877,6 +1007,69 @@ class ExportDistrictRongHe extends BaseBatch{
CcbFtpUtils.ftpUpload('rh', fileName, curDate)
}
def exportPgThirdPartyPrice2(price_date, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujiabds_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} ";
def rets1 = getPgThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} ";
def rets2 = getPgThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujiabds_avprc_dt_val='${price_date}' ";
allRows = getPgThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
index = index + 1
if(index==1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload('rh', fileName, curDate)
}
def getPgThirdPartyDatas(def sql){
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujia_dtsr_id
data << r.gujia_dtsr_dsc
data << r.gujiabds_avprc_dt_val
data << r.gujia_bds_comm_avprc
data << r.gujiabdscomm_hgst_prc
data << r.gujiabdscomm_lwst_prc
data << r.data_udt_tm
allRows << data
})
return allRows;
}
def getThirdPartyDatas(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
@@ -957,6 +1150,49 @@ class ExportDistrictRongHe extends BaseBatch{
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportPgCommunityBasePrice(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${yearMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_bds_clmtd
data << r.gujiabds_fwd_samp_rto
data << r.gujia_bds_mns_grd
data << r.gujia_bds_cfdrg
data << r.gujiabdsarndcmprccrlt
data << r.gujia_bds_cfdc_dgr
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
def headers = []
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
@@ -1031,6 +1267,74 @@ class ExportDistrictRongHe extends BaseBatch{
println "community_p_describe_gj 上传Ftp完成......"
}
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from t_app_p9pha_plda.avm_community_p_describe_gj_h where gujia_bds_mo_val=${month} AND urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_comm_id
data << r.urbn_id
data << r.urbn_nm
data << r.admndiv_ecd
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_als_dsc
data << r.gujia_bds_bss_prc
data << r.gujiabds_cfdc_dgr_grd
data << r.gujiabdsrt3mlgcasenum
data << r.gujiabdslistgclwstprc
data << r.gujiabdslistgcaemdval
data << r.gujiabdslistgcase_avg
data << r.gujiabdslistgcemaxprc
data << r.gujia_bds_mo_val
data << r.gujiabdsavgwminvalavg
data << r.gujiabdsavgwmaxvalavg
data << r.gujia_bds_rel_err_val
data << r.gujia_dvtdgr_dsc
data << r.gujia_als_prc_rng
data << r.gujiaprcminandmax_rng
data << r.gujia_dsc
data << r.gujiabds_lmth_bss_prc
data << r.gujiabds_wthr_mnl_val
data << r.gujiabds_lmth_fnl_prc
data << r.data_crt_tm
data << r.gujiabdsnervetuntwprc
data << r.gujia_prc_src
data << r.gujiabdsabtjlcmbitval
data << r.gujiabdsabtjlgcaseavg
data << r.gujia_indx_lbl
data << r.gujia_prim_pty_tp
data << r.gujia_comm_tp_val
data << r.gujiabds_ltst_mdl_prc
data << r.gujia_ltst_mdl_dt
data << r.gujiabdsidxnmo_moonmo
data << r.gujiabdsa1yiminmmonmo
data << r.gujiabdsidxnyr_moonmo
data << r.gujiabdsat1yspdgrscor
data << r.gujiabdsarnd1kscavprc
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
@@ -1072,6 +1376,75 @@ class ExportDistrictRongHe extends BaseBatch{
}
def exportPgGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "SELECT * FROM t_app_p9pha_plda.avm_guide_statistics where gujia_clc_yrmo='${time}' and urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_model_type
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
data << r.gujia_comm_sum_count
data << r.gujia_comm_count
data << r.gujia_ratio
data << r.gujia_type
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "guide_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "guide_statistics 上传Ftp完成......"
}
def exportPgBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "SELECT * FROM t_app_p9pha_plda.avm_bigdata_supplier_statistics WHERE gujia_clc_yrmo='${time}' and urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
data << r.gujia_data_sour
data << r.gujia_third_comm_count
data << r.gujia_model_type
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_supplier_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "bigdata_supplier_statistics 上传Ftp完成......"
}
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
@@ -1110,6 +1483,7 @@ class ExportDistrictRongHe extends BaseBatch{
println "bigdata_supplier_statistics 上传Ftp完成......"
}
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
@@ -1153,6 +1527,44 @@ class ExportDistrictRongHe extends BaseBatch{
println "bigdata_modeling_base 上传Ftp完成......"
}
def exportPgBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select * from t_app_p9pha_plda.avm_bigdata_modeling_base_gj where gujia_clc_yrmo='${time}' and urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
data << r.gujia_esti_deal_ct
data << r.gujia_esti_comm_ct
data << r.gujia_deal_ct
data << r.gujia_deal_comm_ct
data << r.gujia_sale_ct
data << r.gujia_sale_comm_ct
data << r.gujia_model_type
data << r.gujia_clean_type
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_modeling_base 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "bigdata_modeling_base 上传Ftp完成......"
}
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
@@ -1363,6 +1775,54 @@ class ExportDistrictRongHe extends BaseBatch{
println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def exportPgBigdataForsale(publishDate,endDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and Anc_Tm>='${publishDate}' and Anc_Tm<'${endDate}' and gujia_admndiv_id not in ${newstr}";
def rets1 = getPgHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and Anc_Tm>='${publishDate}' and Anc_Tm<'${endDate}'";
allRows = getPgHouseDatas(sql)
}
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from base.t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
def rets2 = getPgHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true);
}
}
println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def allRows = []
def index = 0
@@ -1432,6 +1892,75 @@ class ExportDistrictRongHe extends BaseBatch{
}
def getPgHouseDatas(def sql){
def allRows = []
def index = 0
pgsqlInstance.eachRow(sql,{r->
index = index+1
println r.urbn_nm +" "+index
def data = []
data << r.urbn_id
data << r.gujia_sell_id
data << r.gujia_listg_hs_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bds_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_bds_unitprc
data << r.gujia_bds_tot_prc
data << r.gujia_sell_pymtmod
data << r.gujia_bds_wbt_floor
data << r.gujiabdstot_floor_num
data << r.gujia_bld_tp_val
data << r.gujia_indr_stc_val
data << r.gujia_bld_stc
data << r.guiabconstructionarea
data << r.gujiabdsset_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_bds_room_num
data << r.gujia_bds_dept_num
data << r.gujia_bds_ktch_num
data << r.gujia_bds_hlt_btw_num
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujiawthrext_lift_ind
data << r.gujiajiladderjiactval
data << r.anc_tm
data << r.exdat
data << r.gujia_wthr_strt
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.gujia_anc_psn
data << r.gujia_anc_psn_tel
data << r.gujia_dsc_inf_cntnt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
// data << r.ATTACH //融合房源没有这个字段
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
});
return allRows
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
@@ -988,6 +988,33 @@ class CommonDatautils {
return ret;
}
public static def findDistrictByParams(String city, String region, String name, String cat, List fields){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("cat", cat);
q.put("region", region)
q.put("name", name);
q.put("status", 0)
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
DSCursor cursor = getDataStoreConnection().find(q, keys);
def dataObj = null
if(cursor){
for(def obj : cursor){
dataObj = obj
break;
}
}
return dataObj;
}
public static def findAllStatusDistrict(String city, String region, String name, String cat, List fields){
DBObject q = new BasicDBObject();
@@ -1,8 +1,10 @@
package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -24,8 +26,21 @@ class CityStatExport {
static main(args) {
def task = new CityStatExport();
task.run()
// task.run()
// task.run1()
// task.regionStat()
task.stat()
}
def stat(){
def fileName = "C:\\Users\\gaojuan\\Desktop\\江苏城市字段信息统计_交付.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['A'] + " "+ d['B']
def rets = CcbCommonUtils.findAllDistrictByCityAndRegion(d['A'], d['B'],"district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
if(rets){
d['H'] = rets.size()
}
})
}
def run(){
@@ -80,6 +95,15 @@ class CityStatExport {
}
def regionStat(){
def city = "无锡"
def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){
def districts = CommonDatautils.findDistricts(city, r, null)
println r +" "+ districts.size()
}
}
def findHouseCount(def city, def startDate){
def total = 0
@@ -32,9 +32,9 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class CZDataBatch extends BaseBatch{
def regionMap = ["江苏":"320000","常州":"320400","天宁":"320402","钟楼":"320404","戚墅堰":"320405","新北":"320411","武进":"320412","溧阳":"320481","金坛":"320482"];
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com")
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
@@ -134,7 +134,7 @@ class CZDataBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
converDistrict(district, statMap, commids);
FzDataUtils.converDistrict(district, statMap, commids);
println totalCount.get()+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
} catch (Exception e) {
e.printStackTrace()
@@ -165,92 +165,6 @@ class CZDataBatch extends BaseBatch{
// executorService.awaitTermination(30, TimeUnit.MINUTES)
}
def converDistrict(def dist, def statMap, def commids){
def map = [:]
//楼盘编号
/* def c_id = czIdMap.get(dist.city+"#"+dist.region+"#"+dist.name)
if(c_id){
map.num_id = c_id;
}else{
map.num_id = dist.get("_id").toString();
} */
map.num_id = dist.get("_id").toString();
map.cat = "district";
// 楼盘名称
map.name = dist.name
map.alias = dist.alias
map.build_date = getBuildDate(dist);
//附属属性
map.build_type = dist.build_type
map.property_life = dist.property_life
map.estate_type2 = dist.estate_type2
map.build_structure = dist.build_structure
// 楼盘均价
def priceObj = statMap.get(dist.region+"#"+dist.name)
if(priceObj && priceObj.avg_price){
map.avg_price = priceObj.avg_price
}
//map.avg_price = getDistrictPrice(dist.city, dist.region, dist.name, dist.address, dist.block);
// 省 市 区
if(dist.city == "常州"){
map.province_code = regionMap.get("江苏");
map.city_code = regionMap.get(dist.city);
map.region_code = regionMap.get(dist.region);
map.province = "江苏";
}
map.city = dist.city;
map.region = dist.region;
if(!commids.contains(map.num_id)){
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2);
// 基础设施
map.facilities_code = "01";
// 居住社区成熟度
map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2);
// 交通条件
map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2);
// 泊车条件 /
map.stop_car_code = getStopCarCode(dist);
// /学区房类型
map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ;
}
// 地址
map.address = dist.address
// 所属区域
map.in_region_code = getRegionCode(dist.region);
//绿化率
map.virescence_percent = getVirescencePercent(dist.virescence_percent, map.build_date);
//容积率
map.cubage_percent = getCubagePercent(dist.cubage_percent, map.build_date);
// 参考租金(1室) 参考租金(2室) 参考租金(3室) 参考租金(4室) 参考租金(5室) 参考租金(5室以上)
map.rents = getRentPrice2(dist.city, dist.region, dist.name, dist.estate_type2, dist.lat2, dist.lng2, dist.cat);
//物业公司
map.estate_company = getEstateCompany(dist.estate_company);
//物业费
map.estate_charge = getEstateCharge(dist.estate_charge);
saveDistrict(map);
}
def saveDistrict(def m){
DBObject q = new BasicDBObject();
q.put("num_id",m.num_id)
def dbo = DIST_COLL.findOne(q);
DBObject udata = new BasicDBObject();
m.each {k,v->
udata.put(k, v);
}
if(dbo){
udata.removeField("num_id")
DIST_COLL.update(q, new BasicDBObject('$set', udata), false, true);
}else{
DIST_COLL.save(udata);
}
}
def shopRun(shops){
@@ -444,668 +358,6 @@ class CZDataBatch extends BaseBatch{
}
//1-主城区:天宁区、钟楼区 2-郊区:武进区、新北区、戚墅堰区 3-县级市:金坛市、溧阳市
def getRegionCode(def region){
if(region.indexOf("天宁")!=-1||region.indexOf("钟楼")!=-1){
return "01";
}else if(region.indexOf("武进")!=-1||region.indexOf("新北")!=-1||region.indexOf("戚墅堰")!=-1){
return "02";
}else if(region.indexOf("金坛")!=-1||region.indexOf("溧阳")!=-1){
return "03";
}else{
return "99";
}
}
def getVirescencePercent(def virescence_percent, def buildDate){
if(virescence_percent){
virescence_percent = virescence_percent.replace("%","");
}
if(virescence_percent){
if(virescence_percent.toFloat()>50){
virescence_percent = "";
}
}
if(!virescence_percent && buildDate){
if(buildDate==1986){
virescence_percent = 25;
}else if(buildDate == 1987){
virescence_percent = 30;
}else if(buildDate>=1990 && buildDate<=2009){
virescence_percent = 25;
}else if(buildDate == 2010){
virescence_percent = 30;
}
}
if(virescence_percent){
virescence_percent = virescence_percent.toDouble()/100;
return new DecimalFormat("######0.00").format(virescence_percent)
}
return virescence_percent;
}
def getCubagePercent(def cubage_percent, def build_date){
if(!cubage_percent && build_date){
if(build_date >=1990 && build_date<=1999){
return "1.49";
}else if(build_date==2000){
return "1.46";
}
}
return cubage_percent;
}
def getBuildDate(def dbo){
def year = DataHandleUtils.findPattern(dbo.finish_building_date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year<=1900){
year = null;
}else{
return year;
}
if(!year){
year = DataHandleUtils.findPattern(dbo.live_date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year<=1900){
year = null;
}else{
return year;
}
if(!year){
def date = DateUtils.formatToYYYYMMDD(dbo.open_date)
year = DataHandleUtils.findPattern(date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year<=1900){
year = null;
}else{
return year;
}
}
}
if(dbo.build_date){
year = DataHandleUtils.findPattern(dbo.build_date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year && year>1900){
return year;
}
}
return year;
}
def getEstateCompany(def company){
if(company){
company = company.replace("---","").replace("·","");
return company;
}
return "";
}
def getEstateCharge(def charge){
if(charge){
if(charge == "0.0"){
return "";
}else{
Pattern pa=Pattern.compile("([\\d\\.]+)")
Matcher ma=pa.matcher(charge)
if(ma.find()){
charge = ma.group(1)
if(charge.toFloat()>=40){
return "";
}else{
if(charge.endsWith(".0")){
return charge.substring(0, charge.indexOf(".0"))
}else{
return charge;
}
}
}else{
return "";
}
}
}
return "";
}
def getRentPrice2(def city, def region, def name, def estate_type2, def lat2, def lng2, def cat){
def month = CommonDatautils.getDistrictRentLastMonth(city, region, name)
def priceMap = [:];
if(month){
priceMap = CommonDatautils.getDistrictRentPrice(city, region, name, month);
}
if(!priceMap){
priceMap = getRoundAvgRentPrice(city,region, name, estate_type2, lat2, lng2, cat);
}
if(priceMap && priceMap.size()>1){
def prices = []
def rooms = [];
for(int i=1;i<=6;i++){
def price = priceMap.get(i);
if(price){
if(price>20000||price<300){
priceMap.remove(i);
}else{
if(i>3 && price<1000){
priceMap.remove(i);
}else{
prices << price
rooms << i;
}
}
}
}
//排序
prices.sort();
rooms.sort();
if(prices.size()>1){
priceMap = [:];
def index = 0;
for(def price : prices){
index++;
priceMap.put(rooms.get(index-1), price);
}
}
filterData(prices, rooms, priceMap);
}
if(priceMap && estate_type2.contains("别墅")){
priceMap.remove(1);
priceMap.remove(2);
if(priceMap.get(4)<2000){
priceMap.remove(4);
}
if(priceMap.get(3)<2000){
priceMap.remove(3);
}
}
// if(!priceMap){
// priceMap = findRegionRentPrice(priceMap, city, region, estate_type2)
// }
//5室及以上价格小于2000去除
if(priceMap){
if(priceMap.get(5)<2000){
priceMap.remove(5);
}
if(priceMap.get(6)<3000){
priceMap.remove(6);
}
}
def newPriceMap = [:]
if(priceMap){
priceMap.each {k,v->
newPriceMap.put(k.toString(),v)
}
}
return newPriceMap;
}
def findRegionRentPrice(def priceMap,def city, def region, def estate_type2){
if(!priceMap){
priceMap = [:]
if(estate_type2=="别墅"){
def room4 = CommonDatautils.getRegionRent(city, region, 4, "别墅");
if(room4){
def v = DataHandleUtils.random(-100, 500) + 100;
priceMap.put(4, room4+v);
}
}else{
def room1 = CommonDatautils.getRegionRent(city, region, 1, "小区");
def room2 = CommonDatautils.getRegionRent(city, region, 2, "小区");
if(room1){
def v = DataHandleUtils.random(-100, 300) + 100;
priceMap.put(1, room1+v);
}
if(room2){
def v = DataHandleUtils.random(-100, 600) + 100;
priceMap.put(2, room2+v);
}
}
}
return priceMap;
}
def getRoundAvgRentPrice(city,region, name, estate_type2, lat2, lng2, cat ){
if(lng2 && lat2){
def lat = lat2/10000000000;
def lng = lng2/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def ret = CommonDatautils.findDistrictsByLatLng(city, cat, map.maxLat*10000000000, map.minLat*10000000000, map.maxLng*10000000000, map.minLng*10000000000, estate_type2);
if(ret){
def room1_sum = 0;
def room1_count = 0;
def room2_sum = 0;
def room2_count = 0;
for(def obj : ret){
def rentMap = CommonDatautils.getDistrictPrice(city, region, name, estate_type2)
if(rentMap){
if(rentMap.get(1)){
room1_sum = room1_sum + rentMap.get(1);
room1_count ++;
}
if(rentMap.get(2)){
room2_sum = room2_sum + rentMap.get(2);
room2_count ++;
}
}
}
def result = [:];
if(room1_count){
def room1_avg_price = (room1_sum/room1_count).toInteger();
if(room1_avg_price){
result.put(1, room1_avg_price);
}
}
if(room2_count){
def room2_avg_price = (room2_sum/room2_count).toInteger();
if(room2_avg_price){
result.put(2, room2_avg_price);
}
}
return result;
}
}
return null;
}
def static filterData(prices, rooms, priceMap){
def mod = [];
for(int j=1;j<prices.size();j++){
def start = prices[j-1];
def end = prices[j];
def val = (end-start)/start;
if(val>=1){
def room = rooms.get(j);
def price = prices.get(j)
priceMap.remove(room);
prices.remove(j);
}
mod << val;
}
println "mod:"+ mod;
return priceMap;
}
//1-周边公交线路≥10条;2-周边公交线路<10条但≥5条;3-周边公交线路<5条但≥2条;4-周边公交线路<2条
def getTrafficCode(def city, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def trafficMap = CommonDatautils.findBusLineCount(city, map.maxLat, map.minLat, map.maxLng, map.minLng);
def count = 0;
if(trafficMap){
count = trafficMap.bus.size()
}
if(count>=10){
return "01";
}else if(count>=5){
return "02";
}else if(count>=2){
return "03";
}else if(count < 2){
return "04";
}
}
return "99";
}
def getStopCarCode(def dbo){
if(dbo.car_site){
if(dbo.car_site=="<br />"||dbo.car_site=="待定"||dbo.car_site.indexOf("暂无")!=-1||dbo.car_site=="详询售楼处"){
dbo.car_site ="";
}
if(dbo.car_site){
return "01";
}
}
if(!dbo.car_site){
def year = DataHandleUtils.findPattern(dbo.build_date, "(\\d+)");
if(!year){
year = DataHandleUtils.findPattern(dbo.finish_building_date, "(\\d+)");
if(!year){
year = DataHandleUtils.findPattern(dbo.live_date, "(\\d+)");
if(!year){
def date = DateUtils.formatToYYYYMMDD(dbo.open_date)
year = DataHandleUtils.findPattern(date, "(\\d+)");
}
}
}
if(year){
year = Integer.parseInt(year);
}
if(year && year >= 1980){
return "01";
}else if(year && year>0 && year<1980){
return "02";
}else{
if(dbo.name.endsWith("期")||dbo.name.indexOf("别墅")!=-1){
return "01";
}else{
if(dbo.build_type && dbo.build_type.indexOf("高层")!=-1){
return "01";
}else if(dbo.estate_type2 && dbo.estate_type2=="别墅"){
return "01";
}else{
def date = CommonDatautils.findHouseBuildDate(dbo.city, dbo.region, dbo.name, "sell")
if(date){
year = DataHandleUtils.findPattern(date, "(\\d+)");
if(year && year >= 1980){
return "01";
}else if(year && year>0 && year<1980){
return "02";
}
}
}
}
}
return "99"
}
def endWithChar = ["村","宿舍","堂","弄"];
for(def s : endWithChar){
if(dbo.name.endsWith(s)){
return "02";
break;
}
}
return "01";
}
def getSchoolCode(def city, def region, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def redius = 3000;
if(city=="常州" && (region=="天宁"|| region=="钟楼")){
redius = 1000;
}
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, redius)
def schools = CommonDatautils.findPomDetail(city, "学校", map.minLat, map.maxLat, map.minLng, map.maxLng)
def school_count = 0;
if(schools){
def names = [];
for(def schoolName : schools){
if(schoolName.contains("小学") && !names.contains(schoolName)){
names << schoolName
}
}
school_count = names.size();
}
if(school_count >= 1){
return 1;
}
return 0;
}
return 0;
}
def getSophisticationCode(def city, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def supermarket_count = CommonDatautils.findPomCount(city, "超市", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(supermarket_count>=5){
return "01";
}else{
map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 2000)
supermarket_count = CommonDatautils.findPomCount(city, "超市", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(supermarket_count >= 5){
return "02";
}else{
return "03";
}
}
}
return "99";
}
def getPublicServiceSetCode(def city, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def hspt_count = CommonDatautils.findPomCount(city, "医院", map.minLat, map.maxLat, map.minLng, map.maxLng)
def bank_count = CommonDatautils.findPomCount(city, "银行", map.minLat, map.maxLat, map.minLng, map.maxLng)
def school_count = CommonDatautils.findPomCount(city, "学校", map.minLat, map.maxLat, map.minLng, map.maxLng)
def supermarket_count = CommonDatautils.findPomCount(city, "超市", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count1 = CommonDatautils.findPomCount(city, "KTV", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count2 = CommonDatautils.findPomCount(city, "电影院", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count3 = CommonDatautils.findPomCount(city, "网吧", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count4 = CommonDatautils.findPomCount(city, "健身中心", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(hspt_count>=1 && bank_count>=1 && school_count>=1 && supermarket_count>=1 && (yl_count1>=1 || yl_count2>=1 || yl_count3>=1||yl_count4>=1)){
return "01";
}else{
map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 2000)
hspt_count = CommonDatautils.findPomCount(city, "医院", map.minLat, map.maxLat, map.minLng, map.maxLng)
bank_count = CommonDatautils.findPomCount(city, "银行", map.minLat, map.maxLat, map.minLng, map.maxLng)
school_count = CommonDatautils.findPomCount(city, "学校", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count1 = CommonDatautils.findPomCount(city, "KTV", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count2 = CommonDatautils.findPomCount(city, "电影院", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count3 = CommonDatautils.findPomCount(city, "网吧", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count4 = CommonDatautils.findPomCount(city, "健身中心", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(hspt_count>=1 && bank_count>=1 && school_count>=1 && supermarket_count>=1 && (yl_count1>=1 || yl_count2>=1 || yl_count3>=1||yl_count4>=1)){
return "02";
}else{
return "03";
}
}
}
return "99";
}
def getDistrictPrice(def city, def region, def name, def address, def block){
def price = CommonDatautils.findDistrictPrice(city, region, name);
if(price){
return price.avg_price;
}else{
// //样例估价
// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
// try{
// JSONObject jsonObject = JSONObject.fromObject(s);
// def map = DataHandleUtils.JsonToMap(jsonObject);
// if(map.code==200){
// if(map.result){
// return map.result.avgPrice;
// }
//
// }
// }catch(Exception e){
//
// }
//分期找总期
if(name.toString().endsWith("期")){
def tempname = name.toString().substring(0, name.toString().indexOf("期")-1)
def districtStat = CommonDatautils.findDistrictPrice(city, region, tempname);
if(districtStat && districtStat.avg_price){
return districtStat.avg_price+150;
}
}
if(name.toString().endsWith("区")){
def tempname = name.toString().substring(0, name.toString().indexOf("区")-1)
def districtStat = CommonDatautils.findDistrictPrice(city, region, tempname);
if(districtStat && districtStat.avg_price){
return districtStat.avg_price+100;
}
}
//周边均价
// price = findAroundAvgPrice(city, address, 100);
// if(price){
// return price.toString()
// }
//板块均价
if(block){
price = CommonDatautils.queryCityRegionPrice(city, region, block);
if(price){
return price.toInteger()
}
}
//区域均价
if(!block||!price){
price = CommonDatautils.queryCityRegionPrice(city, region, null);
if(price){
return price.toInteger()
}
}
}
return null;
}
def getShopPrice(def priceString, def city, def region, def name){
if(priceString){
def avgPrice = priceString.replace("元/㎡","").replace("房源价", "").replace("房源", "");
if(avgPrice.indexOf("-")!=-1){
def array = avgPrice.split("-")
if(array.length>1){
avgPrice = ((array[1].toInteger()+array[0].toInteger())/2).toInteger();
def v = DataHandleUtils.random(-100, 300) + 100;
return avgPrice.toInteger()+v;
}
}else{
def v = DataHandleUtils.random(-100, 300) + 100;
return avgPrice.toDouble().toInteger()+v;
}
}else{
def avgPrice = CommonDatautils.queryShopPrice(city, region, name);
if(avgPrice){
return avgPrice;
}
}
def avgPrice = CommonDatautils.queryRegionBusinessPrice(city, region);
if(avgPrice){
def v = DataHandleUtils.random(-100, 300) + 100;
return avgPrice.toInteger()+v;
}
return null;
}
def getRentNumber(def rentPrice,def city, def region, def name){
if(rentPrice){
if(rentPrice.indexOf("元/㎡/天")!=-1){
rentPrice = rentPrice.replace("元/㎡/天","");
if(rentPrice){
return rentPrice.toDouble();
}
}else if(rentPrice.indexOf("元/㎡/月")!=-1){
def price = rentPrice.replace("元/㎡/月","").toString().trim();
price = price.toFloat();
price = price/30;
return DataHandleUtils.getDoubleRound(2, price.toDouble());
}else{
if(rentPrice.contains("-")){
rentPrice = rentPrice.substring(0, rentPrice.indexOf("-"));
}
return DataHandleUtils.getDoubleRound(2, rentPrice.toDouble());
}
}else{
def day_avg_price = CommonDatautils.queryBusinessDayPrice(city, region, name, "商铺");
if(day_avg_price){
return day_avg_price;
}
}
return null;
}
def getStreetCode(def val){
if(val){
if(val == "无交通管制"){
return 1;
}else if(val == "单行道"){
return 2;
}else if(val == "禁行"){
return 3;
}
return 1;
}
return 1;
}
def getShopEstateTypeCode(def estateType){
if(estateType){
if(estateType.indexOf("商场商铺")!=-1){
return 1;
}else if(estateType.indexOf("专业市场")!=-1){
return 2;
}else if(estateType.indexOf("商业街")!=-1){
return 3;
}else if(estateType.indexOf("社区商铺")!=-1){
return 4;
}
}
return 99;
}
def converList(def dbCursor){
def list = []
@@ -0,0 +1,72 @@
package com.fangjia.script.jiangnanblank
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.bson.types.ObjectId
@BatchOwner(Watcher.GaoJuan)
class DayUpdateDistRefreshBatch extends BaseBatch{
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
public static final DBCollection TASK_COLL = MongoUtils.getCollectionByName("fangjia", "district_update_task", "mongo.db.fangjia.com")
static main(args) {
runBatch(new DayUpdateDistRefreshBatch(), args);
}
@Override
void process(Object args) {
def dists = findAllNeedRefreshDist()
if(dists){
for(def d : dists){
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
if(distObj && distObj._id){
FzDataUtils.converDistrict(distObj, null, null);
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
updateTaskStatus(d.id, 2);
}
}
}else{
println new Date().toString()+ " 无新增小区"
}
}
def updateTaskStatus(def id, def status){
DBObject q = new BasicDBObject()
q.put("_id", new ObjectId(id))
DBObject udata = new BasicDBObject()
udata.put("status", status);
udata.put("m_date", new Date());
TASK_COLL.update(q, new BasicDBObject('$set', udata));
}
def findAllNeedRefreshDist(){
def rets = []
def q = new BasicDBObject()
q.put("status", 5);//未处理
TASK_COLL.find(q).each {d->
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
}
return rets;
}
def converList(def dbCursor){
def list = []
for(def dbo : dbCursor){
list << dbo
}
return list;
}
}
@@ -0,0 +1,781 @@
package com.fangjia.script.jiangnanblank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.text.DecimalFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
class FzDataUtils {
static def regionMap = ["江苏":"320000","常州":"320400","天宁":"320402","钟楼":"320404","戚墅堰":"320405","新北":"320411","武进":"320412","溧阳":"320481","金坛":"320482"];
static DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com")
static def converDistrict(def dist, def statMap, def commids){
def map = [:]
//楼盘编号
/* def c_id = czIdMap.get(dist.city+"#"+dist.region+"#"+dist.name)
if(c_id){
map.num_id = c_id;
}else{
map.num_id = dist.get("_id").toString();
} */
map.num_id = dist.get("_id").toString();
map.cat = "district";
// 楼盘名称
map.name = dist.name
map.alias = dist.alias
map.build_date = getBuildDate(dist);
//附属属性
map.build_type = dist.build_type
map.property_life = dist.property_life
map.estate_type2 = dist.estate_type2
map.build_structure = dist.build_structure
// 楼盘均价
def priceObj = null
if(statMap){
priceObj = statMap.get(dist.region+"#"+dist.name)
}
if(priceObj && priceObj.avg_price){
map.avg_price = priceObj.avg_price
}else{
priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name);
if(priceObj){
map.avg_price = priceObj.avg_price
}
}
// 省 市 区
if(dist.city == "常州"){
map.province_code = regionMap.get("江苏");
map.city_code = regionMap.get(dist.city);
map.region_code = regionMap.get(dist.region);
map.province = "江苏";
}
map.city = dist.city;
map.region = dist.region;
if((commids && !commids.contains(map.num_id)) || !commids){
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2);
// 基础设施
map.facilities_code = "01";
// 居住社区成熟度
map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2);
// 交通条件
map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2);
// 泊车条件 /
map.stop_car_code = getStopCarCode(dist);
// /学区房类型
map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ;
}
// 地址
map.address = dist.address
// 所属区域
map.in_region_code = getRegionCode(dist.region);
//绿化率
map.virescence_percent = getVirescencePercent(dist.virescence_percent, map.build_date);
//容积率
map.cubage_percent = getCubagePercent(dist.cubage_percent, map.build_date);
// 参考租金(1室) 参考租金(2室) 参考租金(3室) 参考租金(4室) 参考租金(5室) 参考租金(5室以上)
map.rents = getRentPrice2(dist.city, dist.region, dist.name, dist.estate_type2, dist.lat2, dist.lng2, dist.cat);
//物业公司
map.estate_company = getEstateCompany(dist.estate_company);
//物业费
map.estate_charge = getEstateCharge(dist.estate_charge);
saveDistrict(map);
}
static def saveDistrict(def m){
DBObject q = new BasicDBObject();
q.put("num_id",m.num_id)
def dbo = DIST_COLL.findOne(q);
DBObject udata = new BasicDBObject();
m.each {k,v->
udata.put(k, v);
}
if(dbo){
udata.removeField("num_id")
DIST_COLL.update(q, new BasicDBObject('$set', udata), false, true);
}else{
DIST_COLL.save(udata);
}
}
//1-主城区:天宁区、钟楼区 2-郊区:武进区、新北区、戚墅堰区 3-县级市:金坛市、溧阳市
static def getRegionCode(def region){
if(region.indexOf("天宁")!=-1||region.indexOf("钟楼")!=-1){
return "01";
}else if(region.indexOf("武进")!=-1||region.indexOf("新北")!=-1||region.indexOf("戚墅堰")!=-1){
return "02";
}else if(region.indexOf("金坛")!=-1||region.indexOf("溧阳")!=-1){
return "03";
}else{
return "99";
}
}
static def getVirescencePercent(def virescence_percent, def buildDate){
if(virescence_percent){
virescence_percent = virescence_percent.replace("%","");
}
if(virescence_percent){
if(virescence_percent.toFloat()>50){
virescence_percent = "";
}
}
if(!virescence_percent && buildDate){
if(buildDate==1986){
virescence_percent = 25;
}else if(buildDate == 1987){
virescence_percent = 30;
}else if(buildDate>=1990 && buildDate<=2009){
virescence_percent = 25;
}else if(buildDate == 2010){
virescence_percent = 30;
}
}
if(virescence_percent){
virescence_percent = virescence_percent.toDouble()/100;
return new DecimalFormat("######0.00").format(virescence_percent)
}
return virescence_percent;
}
static def getCubagePercent(def cubage_percent, def build_date){
if(!cubage_percent && build_date){
if(build_date >=1990 && build_date<=1999){
return "1.49";
}else if(build_date==2000){
return "1.46";
}
}
return cubage_percent;
}
static def getBuildDate(def dbo){
def year = DataHandleUtils.findPattern(dbo.finish_building_date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year<=1900){
year = null;
}else{
return year;
}
if(!year){
year = DataHandleUtils.findPattern(dbo.live_date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year<=1900){
year = null;
}else{
return year;
}
if(!year){
def date = DateUtils.formatToYYYYMMDD(dbo.open_date)
year = DataHandleUtils.findPattern(date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year<=1900){
year = null;
}else{
return year;
}
}
}
if(dbo.build_date){
year = DataHandleUtils.findPattern(dbo.build_date, "(\\d+)");
if(year){
year = Integer.parseInt(year);
}
if(year && year>1900){
return year;
}
}
return year;
}
static def getEstateCompany(def company){
if(company){
company = company.replace("---","").replace("·","");
return company;
}
return "";
}
static def getEstateCharge(def charge){
if(charge){
if(charge == "0.0"){
return "";
}else{
Pattern pa=Pattern.compile("([\\d\\.]+)")
Matcher ma=pa.matcher(charge)
if(ma.find()){
charge = ma.group(1)
if(charge.toFloat()>=40){
return "";
}else{
if(charge.endsWith(".0")){
return charge.substring(0, charge.indexOf(".0"))
}else{
return charge;
}
}
}else{
return "";
}
}
}
return "";
}
static def getRentPrice2(def city, def region, def name, def estate_type2, def lat2, def lng2, def cat){
def month = CommonDatautils.getDistrictRentLastMonth(city, region, name)
def priceMap = [:];
if(month){
priceMap = CommonDatautils.getDistrictRentPrice(city, region, name, month);
}
if(!priceMap){
priceMap = getRoundAvgRentPrice(city,region, name, estate_type2, lat2, lng2, cat);
}
if(priceMap && priceMap.size()>1){
def prices = []
def rooms = [];
for(int i=1;i<=6;i++){
def price = priceMap.get(i);
if(price){
if(price>20000||price<300){
priceMap.remove(i);
}else{
if(i>3 && price<1000){
priceMap.remove(i);
}else{
prices << price
rooms << i;
}
}
}
}
//排序
prices.sort();
rooms.sort();
if(prices.size()>1){
priceMap = [:];
def index = 0;
for(def price : prices){
index++;
priceMap.put(rooms.get(index-1), price);
}
}
filterData(prices, rooms, priceMap);
}
if(priceMap && estate_type2.contains("别墅")){
priceMap.remove(1);
priceMap.remove(2);
if(priceMap.get(4)<2000){
priceMap.remove(4);
}
if(priceMap.get(3)<2000){
priceMap.remove(3);
}
}
// if(!priceMap){
// priceMap = findRegionRentPrice(priceMap, city, region, estate_type2)
// }
//5室及以上价格小于2000去除
if(priceMap){
if(priceMap.get(5)<2000){
priceMap.remove(5);
}
if(priceMap.get(6)<3000){
priceMap.remove(6);
}
}
def newPriceMap = [:]
if(priceMap){
priceMap.each {k,v->
newPriceMap.put(k.toString(),v)
}
}
return newPriceMap;
}
static def findRegionRentPrice(def priceMap,def city, def region, def estate_type2){
if(!priceMap){
priceMap = [:]
if(estate_type2=="别墅"){
def room4 = CommonDatautils.getRegionRent(city, region, 4, "别墅");
if(room4){
def v = DataHandleUtils.random(-100, 500) + 100;
priceMap.put(4, room4+v);
}
}else{
def room1 = CommonDatautils.getRegionRent(city, region, 1, "小区");
def room2 = CommonDatautils.getRegionRent(city, region, 2, "小区");
if(room1){
def v = DataHandleUtils.random(-100, 300) + 100;
priceMap.put(1, room1+v);
}
if(room2){
def v = DataHandleUtils.random(-100, 600) + 100;
priceMap.put(2, room2+v);
}
}
}
return priceMap;
}
static def getRoundAvgRentPrice(city,region, name, estate_type2, lat2, lng2, cat ){
if(lng2 && lat2){
def lat = lat2/10000000000;
def lng = lng2/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def ret = CommonDatautils.findDistrictsByLatLng(city, cat, map.maxLat*10000000000, map.minLat*10000000000, map.maxLng*10000000000, map.minLng*10000000000, estate_type2);
if(ret){
def room1_sum = 0;
def room1_count = 0;
def room2_sum = 0;
def room2_count = 0;
for(def obj : ret){
def rentMap = CommonDatautils.getDistrictPrice(city, region, name, estate_type2)
if(rentMap){
if(rentMap.get(1)){
room1_sum = room1_sum + rentMap.get(1);
room1_count ++;
}
if(rentMap.get(2)){
room2_sum = room2_sum + rentMap.get(2);
room2_count ++;
}
}
}
def result = [:];
if(room1_count){
def room1_avg_price = (room1_sum/room1_count).toInteger();
if(room1_avg_price){
result.put(1, room1_avg_price);
}
}
if(room2_count){
def room2_avg_price = (room2_sum/room2_count).toInteger();
if(room2_avg_price){
result.put(2, room2_avg_price);
}
}
return result;
}
}
return null;
}
def static filterData(prices, rooms, priceMap){
def mod = [];
for(int j=1;j<prices.size();j++){
def start = prices[j-1];
def end = prices[j];
def val = (end-start)/start;
if(val>=1){
def room = rooms.get(j);
def price = prices.get(j)
priceMap.remove(room);
prices.remove(j);
}
mod << val;
}
println "mod:"+ mod;
return priceMap;
}
//1-周边公交线路≥10条;2-周边公交线路<10条但≥5条;3-周边公交线路<5条但≥2条;4-周边公交线路<2条
static def getTrafficCode(def city, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def trafficMap = CommonDatautils.findBusLineCount(city, map.maxLat, map.minLat, map.maxLng, map.minLng);
def count = 0;
if(trafficMap){
count = trafficMap.bus.size()
}
if(count>=10){
return "01";
}else if(count>=5){
return "02";
}else if(count>=2){
return "03";
}else if(count < 2){
return "04";
}
}
return "99";
}
static def getStopCarCode(def dbo){
if(dbo.car_site){
if(dbo.car_site=="<br />"||dbo.car_site=="待定"||dbo.car_site.indexOf("暂无")!=-1||dbo.car_site=="详询售楼处"){
dbo.car_site ="";
}
if(dbo.car_site){
return "01";
}
}
if(!dbo.car_site){
def year = DataHandleUtils.findPattern(dbo.build_date, "(\\d+)");
if(!year){
year = DataHandleUtils.findPattern(dbo.finish_building_date, "(\\d+)");
if(!year){
year = DataHandleUtils.findPattern(dbo.live_date, "(\\d+)");
if(!year){
def date = DateUtils.formatToYYYYMMDD(dbo.open_date)
year = DataHandleUtils.findPattern(date, "(\\d+)");
}
}
}
if(year){
year = Integer.parseInt(year);
}
if(year && year >= 1980){
return "01";
}else if(year && year>0 && year<1980){
return "02";
}else{
if(dbo.name.endsWith("期")||dbo.name.indexOf("别墅")!=-1){
return "01";
}else{
if(dbo.build_type && dbo.build_type.indexOf("高层")!=-1){
return "01";
}else if(dbo.estate_type2 && dbo.estate_type2=="别墅"){
return "01";
}else{
def date = CommonDatautils.findHouseBuildDate(dbo.city, dbo.region, dbo.name, "sell")
if(date){
year = DataHandleUtils.findPattern(date, "(\\d+)");
if(year && year >= 1980){
return "01";
}else if(year && year>0 && year<1980){
return "02";
}
}
}
}
}
return "99"
}
def endWithChar = ["村","宿舍","堂","弄"];
for(def s : endWithChar){
if(dbo.name.endsWith(s)){
return "02";
break;
}
}
return "01";
}
static def getSchoolCode(def city, def region, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def redius = 3000;
if(city=="常州" && (region=="天宁"|| region=="钟楼")){
redius = 1000;
}
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, redius)
def schools = CommonDatautils.findPomDetail(city, "学校", map.minLat, map.maxLat, map.minLng, map.maxLng)
def school_count = 0;
if(schools){
def names = [];
for(def schoolName : schools){
if(schoolName.contains("小学") && !names.contains(schoolName)){
names << schoolName
}
}
school_count = names.size();
}
if(school_count >= 1){
return 1;
}
return 0;
}
return 0;
}
static def getSophisticationCode(def city, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def supermarket_count = CommonDatautils.findPomCount(city, "超市", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(supermarket_count>=5){
return "01";
}else{
map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 2000)
supermarket_count = CommonDatautils.findPomCount(city, "超市", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(supermarket_count >= 5){
return "02";
}else{
return "03";
}
}
}
return "99";
}
static def getPublicServiceSetCode(def city, def lat, def lng){
if(lng && lat){
lat = lat/10000000000;
lng = lng/10000000000;
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
def hspt_count = CommonDatautils.findPomCount(city, "医院", map.minLat, map.maxLat, map.minLng, map.maxLng)
def bank_count = CommonDatautils.findPomCount(city, "银行", map.minLat, map.maxLat, map.minLng, map.maxLng)
def school_count = CommonDatautils.findPomCount(city, "学校", map.minLat, map.maxLat, map.minLng, map.maxLng)
def supermarket_count = CommonDatautils.findPomCount(city, "超市", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count1 = CommonDatautils.findPomCount(city, "KTV", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count2 = CommonDatautils.findPomCount(city, "电影院", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count3 = CommonDatautils.findPomCount(city, "网吧", map.minLat, map.maxLat, map.minLng, map.maxLng)
def yl_count4 = CommonDatautils.findPomCount(city, "健身中心", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(hspt_count>=1 && bank_count>=1 && school_count>=1 && supermarket_count>=1 && (yl_count1>=1 || yl_count2>=1 || yl_count3>=1||yl_count4>=1)){
return "01";
}else{
map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 2000)
hspt_count = CommonDatautils.findPomCount(city, "医院", map.minLat, map.maxLat, map.minLng, map.maxLng)
bank_count = CommonDatautils.findPomCount(city, "银行", map.minLat, map.maxLat, map.minLng, map.maxLng)
school_count = CommonDatautils.findPomCount(city, "学校", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count1 = CommonDatautils.findPomCount(city, "KTV", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count2 = CommonDatautils.findPomCount(city, "电影院", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count3 = CommonDatautils.findPomCount(city, "网吧", map.minLat, map.maxLat, map.minLng, map.maxLng)
yl_count4 = CommonDatautils.findPomCount(city, "健身中心", map.minLat, map.maxLat, map.minLng, map.maxLng)
if(hspt_count>=1 && bank_count>=1 && school_count>=1 && supermarket_count>=1 && (yl_count1>=1 || yl_count2>=1 || yl_count3>=1||yl_count4>=1)){
return "02";
}else{
return "03";
}
}
}
return "99";
}
static def getDistrictPrice(def city, def region, def name, def address, def block){
def price = CommonDatautils.findDistrictPrice(city, region, name);
if(price){
return price.avg_price;
}else{
// //样例估价
// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
// try{
// JSONObject jsonObject = JSONObject.fromObject(s);
// def map = DataHandleUtils.JsonToMap(jsonObject);
// if(map.code==200){
// if(map.result){
// return map.result.avgPrice;
// }
//
// }
// }catch(Exception e){
//
// }
//分期找总期
if(name.toString().endsWith("期")){
def tempname = name.toString().substring(0, name.toString().indexOf("期")-1)
def districtStat = CommonDatautils.findDistrictPrice(city, region, tempname);
if(districtStat && districtStat.avg_price){
return districtStat.avg_price+150;
}
}
if(name.toString().endsWith("区")){
def tempname = name.toString().substring(0, name.toString().indexOf("区")-1)
def districtStat = CommonDatautils.findDistrictPrice(city, region, tempname);
if(districtStat && districtStat.avg_price){
return districtStat.avg_price+100;
}
}
//周边均价
// price = findAroundAvgPrice(city, address, 100);
// if(price){
// return price.toString()
// }
//板块均价
if(block){
price = CommonDatautils.queryCityRegionPrice(city, region, block);
if(price){
return price.toInteger()
}
}
//区域均价
if(!block||!price){
price = CommonDatautils.queryCityRegionPrice(city, region, null);
if(price){
return price.toInteger()
}
}
}
return null;
}
static def getShopPrice(def priceString, def city, def region, def name){
if(priceString){
def avgPrice = priceString.replace("元/㎡","").replace("房源价", "").replace("房源", "");
if(avgPrice.indexOf("-")!=-1){
def array = avgPrice.split("-")
if(array.length>1){
avgPrice = ((array[1].toInteger()+array[0].toInteger())/2).toInteger();
def v = DataHandleUtils.random(-100, 300) + 100;
return avgPrice.toInteger()+v;
}
}else{
def v = DataHandleUtils.random(-100, 300) + 100;
return avgPrice.toDouble().toInteger()+v;
}
}else{
def avgPrice = CommonDatautils.queryShopPrice(city, region, name);
if(avgPrice){
return avgPrice;
}
}
def avgPrice = CommonDatautils.queryRegionBusinessPrice(city, region);
if(avgPrice){
def v = DataHandleUtils.random(-100, 300) + 100;
return avgPrice.toInteger()+v;
}
return null;
}
static def getRentNumber(def rentPrice,def city, def region, def name){
if(rentPrice){
if(rentPrice.indexOf("元/㎡/天")!=-1){
rentPrice = rentPrice.replace("元/㎡/天","");
if(rentPrice){
return rentPrice.toDouble();
}
}else if(rentPrice.indexOf("元/㎡/月")!=-1){
def price = rentPrice.replace("元/㎡/月","").toString().trim();
price = price.toFloat();
price = price/30;
return DataHandleUtils.getDoubleRound(2, price.toDouble());
}else{
if(rentPrice.contains("-")){
rentPrice = rentPrice.substring(0, rentPrice.indexOf("-"));
}
return DataHandleUtils.getDoubleRound(2, rentPrice.toDouble());
}
}else{
def day_avg_price = CommonDatautils.queryBusinessDayPrice(city, region, name, "商铺");
if(day_avg_price){
return day_avg_price;
}
}
return null;
}
static def getStreetCode(def val){
if(val){
if(val == "无交通管制"){
return 1;
}else if(val == "单行道"){
return 2;
}else if(val == "禁行"){
return 3;
}
return 1;
}
return 1;
}
static def getShopEstateTypeCode(def estateType){
if(estateType){
if(estateType.indexOf("商场商铺")!=-1){
return 1;
}else if(estateType.indexOf("专业市场")!=-1){
return 2;
}else if(estateType.indexOf("商业街")!=-1){
return 3;
}else if(estateType.indexOf("社区商铺")!=-1){
return 4;
}
}
return 99;
}
}
@@ -498,7 +498,7 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
names.add(ClientUtils.escapeQueryChars(name));
if(alias != null){
for(String alia : alias){
if(StringUtils.isNotBlank(alia) && alia.length() > 2 && !names.contains(alia)){
if(StringUtils.isNotBlank(alia) && alia.length() >= 2 && !names.contains(alia)){
names.add(ClientUtils.escapeQueryChars(alia));
}
}