This commit is contained in:
2021-10-18 15:48:53 +08:00
175 changed files with 12218 additions and 843 deletions
@@ -13,8 +13,9 @@ import com.mongodb.DBObject
*/
class CityLatlngBox {
public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017)
//public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020)
public static void main(String[] args) {
CityLatlngBox sd = new CityLatlngBox();
@@ -22,7 +23,7 @@ class CityLatlngBox {
}
void process() {
def cities = CommonUtils.findAllCity()
def cities = ["白沙","阿坝州","甘孜州","阿勒泰","临沧","那曲","三沙","喀什","昌吉","迪庆州","普洱","德宏州","湘西","塔城","凉山州","屯昌","山南","海西州","阿拉善盟","乐东","定安","怒江州","黄山","图木舒克","楚雄","临高","乌兰察布","延边","文山州","锡林郭勒盟","陵水","天门","淮北","黔东南","万宁","湘潭","海北州","琼中","吐鲁番","惠州","巴音郭楞","恩施","铜仁","黔南","西双版纳州","潜江","林芝","文昌","仙桃","神农架","和田","黔西南","果洛州","儋州","哈密","大理","东方","日喀则","兴安盟","昌江","澄迈","保亭","济源","五指山","阿拉尔","伊犁","临夏","海南州","琼海","石河子","昌都","五家渠","玉树州","博尔塔拉","东营","铜陵","黄南州","廊坊","嘉峪关","马鞍山","宁波","广安","中山","克孜勒苏","厦门","莱芜","鄂州","东莞","海口","鹤壁","六盘水","漯河","三亚","珠海","深圳","乌海","淮南","北海","双鸭山","襄阳","新余","汕头","克拉玛依","鹰潭","盘锦","镇江","石嘴山","嘉兴","铜川","焦作","芜湖","潮州","萍乡","红河","黄石","辽源","辽阳","枣庄","毕节","许昌","阳泉","自贡","遂宁","景德镇","营口","日照","莆田"]//CommonUtils.findAllCity()
for (String city : cities) {
println city
BOUNDS_COLL.remove(new BasicDBObject('city', city))
@@ -70,13 +71,13 @@ class CityLatlngBox {
int row = 0
int inx = 0
def buffer = []
while(seedLatBD < cityBoundBD.getMaxLat()) {
while(seedLatGD < cityBoundGD.getMaxLat()) {
int col = 0
Bounds boundsBD = null;
Bounds boundsQQ = null;
Bounds boundsGD = null;
while (seedLngBD < cityBoundBD.getMaxLng()) {
while (seedLngGD < cityBoundGD.getMaxLng()) {
double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // 右侧OFFSET处的经纬度
double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // 上方OFFSET处的经纬度
@@ -0,0 +1,88 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 建行模型估值项目
* 小区建筑类型数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
runBatch(new BigdataCommbuildtypeBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rets = []
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def rows = createDatas(cityCodeDatas.get(city), dists)
rets.addAll(rows)
println city +" "+rows.size()
}
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
println "txt 生成"
}
def createDatas(cityCode, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCode, dist)
if(ret){
rets << ret
}
println dist.city +" "+dist.region+" "+dist.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(cityCode, dist){
def ret = []
ret << cityCode
ret << dist._id.toString()
ret << CcbCodeUtils.parseBuildType(dist.build_type)
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
}
@@ -0,0 +1,216 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 小区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" 小区量:"+dists.size()
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println city +" "+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
}
def allDatas = []
allDatas << headers
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
if(!fangjiaDists.contains(ccbId)){
def datas = ccbMaps.get(ccbId)
datas[datas.size()-1] = 1
rets << datas
}
}
println "添加删除状态小区:"+rets.size()
return rets
}
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
//def rets = Collections.synchronizedList(new ArrayList());
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDataMap, distCodeMap, dist)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(def cityCodeDataMap, def distCodeMap, def dist){
def ret = []
try{
def data = distCodeMap.get(dist.region)
if(!data){
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
return ret;
}
ret << cityCodeDataMap.get(dist.city)
if(data){
ret << data.district_code
}else{
ret << ""
}
ret << dist._id.toString()
if(data){
ret << data.district
}else{
ret << ""
}
ret << dist.name
ret << dist.address
if(dist.lat){
def lat = dist.lat/10000000000d
def lng = dist.lng/10000000000d
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << dist.all_area
ret << dist.build_area
ret << dist.virescence_percent
ret << dist.cubage_percent
ret << dist.car_site_count
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "pa100"
ret << ""
if(dist.address){
if(dist.address.toString().contains("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = null
if(dist.address){
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
}
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
ret << ""
}
ret << 0
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << ""
ret << ""
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name
e.printStackTrace()
}
return ret;
}
}
@@ -0,0 +1,543 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 成交数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = 20210701//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new BigdataDealBatch(), args);
}
@Override
public void process(Object args) {
def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
println citys.unique().size()
def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){
// ppeDists = getPpeGTE15Datas(city)
// }
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
def result = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}
}else{
def ret = converData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
ret << number
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
ret << ""
}
ret << trade.dist_id.toString()
ret << trade.master_name
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
ret << ""//期数
ret << trade.floor
ret << ""
ret << trade.height
ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.avg_price
ret << trade.total_price
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
def root = sourceMap.get(trade.source)
if(root){
if(CcbCommonUtils.rootMap.get(root)){
ret << CcbCommonUtils.rootMap.get(root)
}else{
ret << "99"
}
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.total_price
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def getHouseRoom(def room, def hall, def toilet){
def s = ""
if(room){
s = s + room +"室"
}
if(hall){
s = s + hall +"厅"
}
if(toilet){
s = s + toilet +"卫"
}
return s;
}
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = null;
// if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
// if(price){
// if(dist.city=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else if(dist.city=="杭州"){
// startPrice = (int)(price - price*0.15)
// endPrice = (int)(price*0.15 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// }
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
if(!trades && dist.address){
def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)
}
}
if(trades){
// def unitPrice= ppeDistsMap.get(dist._id.toString())
// if(unitPrice){
// for(def t : trades){
// t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
// }
// }
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return tradesList
}
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
//def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
// q.put("premisesName" , "呼玛二村")
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
def index = 0
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
rets << obj;
// def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
// if(price){
// def startPrice = 0
// def endPrice= 0
// if(obj.fjCity=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// if(obj.price>=startPrice && obj.price<=endPrice){
// rets << obj;
// }else{
// index++
// println index + " "+obj
// }
// }
}
}
return rets;
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
def getPpeGTE15Datas(def city){
def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " +
"and cityname='"+city+"'";
def map = [:]
try{
sqlInstance.eachRow(sql,{
map.put(it.commid, it.unit_price)
});
}catch(Exception e){
getPpeGTE15Datas(city)
}
return map
}
}
@@ -0,0 +1,92 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 建行模型估值项目
* 区域数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataDistrictBatch extends BaseBatch{
static main(args) {
runBatch(new BigdataDistrictBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def datas = findDistrictCodeValue(city)
if(datas){
rows.addAll(datas)
}
println city+" "+ datas.size()
}
def allDatas = createDatas(rows)
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
}
def createDatas(rows){
def allRows = []
def index = 1
rows.each { m ->
def data = []
data << m.dist_code
data << m.dist_name
data << m.dist_spell
data << m.city_id
data << m.dist_code
data << index
data << ""
data << ""
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
data << ""
data << ""
data << ""
data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
index++
allRows << data
}
return allRows;
}
def findDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
// q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
def lists = []
for(def db : dbcursor){
if(db.district_code){
def m = [:]
m.id = db._id.toString()
m.dist_name = db.district
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
m.city_id = db.city_code
m.dist_code = db.district_code
lists << m
}
}
return lists;
}
}
@@ -0,0 +1,607 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 出售房源数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataForsaleBatch extends BaseBatch{
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def index = 0
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200101, 20210930);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret && ret.size()<100){
ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
}
if(ret){
def areaData = distCodeMap.get(rMap.region)
if(areaData){
def regionCode = areaData.district_code
def cityCode = cityCodeDataMap.get(city)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
}
}else{
println city +" "+rMap.region+" 没有找到对应的区域"
}
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
}
println city + " 房源总量:"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = []
if(index == 1){
headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
}
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
}
def createSellDatas(def city, def values){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(values.size())
println city +"总房源量:"+values.size()
def datas = Collections.synchronizedList(new ArrayList());
for(def i=0;i<values.size();i++){
final def m = values[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
datas << converData(m)
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return datas ;
}
def converData(def m){
def data = []
data << m.city_code
data << m.houseId
data << ""
data << m.city
data << m.region_code
data << m.region
data << m.district_id
data << m.district_name
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.address==null?"":m.address==null
data << m.build_year==null?"":m.build_year
data << m.proptype==null?"":m.proptype
data << m.propyears==null?"":m.propyears
data << m.avg_price==null?"":m.avg_price
data << m.total_price==null?"":m.total_price
data << ""
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.house_structure_code==null?"":m.house_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
data << m.house_type==null?"":m.house_type
data << m.room==null?"":m.room
data << m.hall==null?"":m.hall
data << ""
data << m.toilet==null?"":m.toilet
data << m.direction_code==null?"":m.direction_code
data << m.fitment_code==null?"":m.fitment_code
data << ""
data << ""
data << m.p_date==null?"":m.p_date
data << ""
data << ""
data << m.root
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.c_date==null?"":m.c_date
data << m.m_date==null?"":m.m_date
data << m.source
data << ""
return data
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = city
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getPublicDate(def pDate){
if(pDate>=20181001 && pDate<20200201){
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
}
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
return pdate;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
private String csvHandlerStr(String str) {
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
String tempDescription=str;
//如果有逗号
if(str.contains(",")||str.contains(",")){
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
if(str.contains("\"")){
// tempDescription=str.replace("\"", "\"\"");
tempDescription=str.replace("\"", "");
}
//在将逗号转义
tempDescription="\""+tempDescription+"\"";
}
return tempDescription;
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
/*
* 生成一个用不重复的ID
*/
def GenNonDuplicateID(randomLength){
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
}
def marge(){
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
def ids = new HashSet();
def index = 0
file.eachLine{it->
index++
if(index>1){
def arrs = it.split("\\|@\\|")
def id = arrs[1]
if(!ids.contains(id)){
ids << id
}
}
}
println index
println ids.size()
}
def getDistrictIds(def cityid){
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
def distIds = new HashSet()
sqlInstance.eachRow(sql,{it->
distIds << it.commid
});
return distIds;
}
}
@@ -0,0 +1,293 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行数据poi同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataPoiBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
runBatch(new BigdataPoiBatch(), args);
}
@Override
public void process(Object args) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
def rets = []
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows1)
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows3)
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_POI_GJ.dat"
}
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "txt 生成"
// def headers1 = ["城市ID","城市","行政区ID","区域名称","经度","纬度","创建时间","来源公司"]
// def regionCenterDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\行政区中心点经纬度.xlsx", 0,1)
// def regionRets = createRegionCenters(regionCenterDatas)
// FileUtils.batchCreateFile(regionRets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\region_center.txt", headers1, "utf-8", false);
// println "行政区域中心txt 生成"
}
def createRegionCenters(def datas){
def rets = []
for(def d : datas){
def ret = []
ret << d['A'].toInteger().toString()
ret << d['B']
ret << d['C'].toInteger().toString()
ret << d['D']
def loactionStr = d['E']
if(loactionStr){
def arrs = loactionStr.split(",")
def loactionMap = GeoUtils.gcjToGps84(arrs[1].toDouble(), arrs[0].toDouble())
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
}else{
ret << ""
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
return rets;
}
def createDatas(citys, cityCodeDataMap, datas, type){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def cityDistMap = CcbCommonUtils.findAllCityDistrictCodeValue()
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(citys.contains(d['D'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){
rets << ret
}
}
println totalCount.get()+" "+ d['D']+" "+d['F']
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
if(jsonObject && jsonObject.info=="OK"){
def compent = jsonObject.regeocode.addressComponent
if(compent){
def district = compent.district
return district;
}
}
return null
}
def addValue2(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['E'])
ret << citycode
def region =getRegionByLocation(d['H'], d['I'])
def distCodeMap = cityDistMap.get(d['E'])
if(region){
region = RegionData.formatRegion(d['E'], region)
}
if(region){
def distObj = distCodeMap.get(region)
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['E']+" " + region+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
if(d['A']){
ret << d['A']
}else{
ret << new ObjectId()
}
ret << d['G']
ret << type
if(type=="地铁"){
ret << "cz125"
}else if(type=="公交"){
ret << "cz123"
}else if(type=="医院"){
ret << "hp111"
}else if(type == "银行"){
ret << "se118"
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['B'].contains("141202")){
ret << "ed138"
}else if(d['B'].contains("141203")){
ret << "ed114"
}
}
if(d['I']){
def lat = d['I'].toDouble()
def lng = d['H'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << d['C']
ret << d['F']
ret << "FJW"
return ret;
}
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['D'])
ret << citycode
def distCodeMap = cityDistMap.get(d['D'])
def region =RegionData.formatRegion(d['D'], d['E'])
if(region){
def distObj = distCodeMap.get(region)
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << d['A']
ret << d['F']
ret << type
if(type=="地铁"){
ret << "cz125"
}else if(type=="公交"){
ret << "cz123"
}else if(type=="医院"){
ret << "hp111"
}else if(type == "银行"){
ret << "se118"
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['F'].contains("中学")){
ret << "ed138"
}else if(d['F'].contains("小学")){
ret << "ed114"
}
}
if(d['I']){
def lat = d['I'].toDouble()
def lng = d['H'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << ""
ret << ""
ret << "FJW"
return ret;
}
}
@@ -0,0 +1,324 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com")
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static DSCollection dataStoreConnection = null;
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def findCityCodeValue(){
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
return cityCodeMap;
}
public static def findAllDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
//q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
map.put(db.fangjia_district, db)
}
return map;
}
public static def findAllCityDistrictCodeValue(city){
def q = new BasicDBObject()
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
def distMap = map.get(db.fangjia_city)
if(!distMap){
distMap = [:]
map.put(db.fangjia_city, distMap)
}
distMap.put(db.fangjia_district, db)
}
return map;
}
public static def findPersonPrice(city, date){
def q = new BasicDBObject()
q.put("city", city)
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
public static def findGuidePrice(city){
def q = new BasicDBObject()
q.put("city", city)
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(dist.name.endsWith("弄小区") || dist.name.endsWith("小区") ){
def name = dist.name.replace("弄小区","弄").replace("小区","")
if(!alias.contains(name)){
alias << name
}
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
// if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
// }
}
return lists
}
public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
q.put("district_name", dist.address)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
}
}
return lists
}
public static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
public static DSCollection getDataStoreConnection(){
if(dataStoreConnection != null) {
return dataStoreConnection
}
DataStore ds = new DataStore("seaweed.db.fangjia.com", 27007)
dataStoreConnection = ds.getCollection("fangjia.seaweed")
return dataStoreConnection
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","御海阳光")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
rets << d
}
}
return rets;
}
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def d : cursor){
if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
return d;
}
}
}
return null;
}
public static def findCcbDistrict(cityid){
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where cityId=${cityid}";
def map = [:]
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
map.put(r.COMMID, data)
});
return map
}
}
@@ -0,0 +1,58 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import org.apache.commons.codec.digest.DigestUtils
class CcbFtpUtils {
static def ftpServer = "192.168.0.45"
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "0I1k8EAM+O8iA"
static def ftpPath = "/home/ccb/output/"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
static def ftpUpload(outputPath_dist, month){
//上传到ftp
sftpChannel = reConnect();
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
}
static def reConnect(){
if(sftpChannel == null){
println "sftpChannel 为空,需重连....."
if(sftp == null){
sftp = new SFTP();
}
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
}else{
println "sftpChannel 正常....."
}
return sftpChannel;
}
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
def data = new FileInputStream(new File(dataFileName))
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static main(args) {
}
}
@@ -0,0 +1,100 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
/**
* 建行模型估值项目
* 城市数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class CityBatch extends BaseBatch{
static main(args) {
runBatch(new CityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def index = 1
for(def city : citys){
def data = CcbCommonUtils.findCityCodeValue()
def map = [:]
map.name = city
if(data){
map.city_code = data.city_code
map.prov_code = data.province_code
}
def cityObj = findCityInfo(city)
if(cityObj){
map.city_spell = cityObj.spell
map.city_level = cityObj.city_grade
map.seq = index
map.ifcapital = isCapital(cityObj.province, city)
map.ifmunicipal = ifMunicipal(city)
}
index++
rows << map
}
def allDatas = createDatas(rows)
def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"]
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false);
}
def createDatas(rows){
def allRows = []
rows.each { m ->
def data = []
data << m.city_code
data << m.name
data << m.city_spell
data << m.prov_code
data << m.city_code
data << m.city_level
data << m.seq
data << CcbCodeUtils.parseFlag(m.ifcapital)
data << CcbCodeUtils.parseFlag(m.ifmunicipal)
data << ""
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "fjw"
allRows << data
}
return allRows;
}
def findCityInfo(city){
def q = new BasicDBObject()
q.put("display", city)
return CcbCommonUtils.SERVER_AREA_COLL.findOne(q)
}
def isCapital(def prov, def city){
def provCity = CcbCommonUtils.provCityMap.get(prov)
if(provCity && provCity==city){
return "是"
}
return "否"
}
def ifMunicipal(def city){
if(["上海", "北京", "天津", "重庆"].contains(city)){
return "是"
}
return "否"
}
}
@@ -0,0 +1,262 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class DistrictDelete {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new DistrictDelete()
// task.run()
// task.gzRun();
// task.deleteDistrict()
task.deleteCommPrice()
}
def run(){
//"泉州","深圳", "重庆",
def citys = ["广州"];//"广州",
def delIndex = 0
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
checkDist(city, dists)
}
}
def checkDist(city,dists){
def rets = Collections.synchronizedList(new ArrayList());
def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd")
def totalCount = new AtomicLong()
def delIndex = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
println city +" 小区量:"+dists.size()
def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"]
for(def i=0;i<dists.size();i++) {
final def dist = dists[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def count = findHouses(city, dist.region, "sell", 20190101,20211231, dist.name, dist.alias)
def count = findHouseCount(city, dist.region, dist.name, dist.alias)
def tradeCount = 0
if(!count){
tradeCount = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, 0, 300000)
if(!tradeCount){
def nameFlag = false
for(def key : keys){
if(dist.name.contains(key)){
nameFlag = true
break;
}
}
if(!nameFlag){
if(dist.name.size()<=3 && (dist.name.contains("村") || dist.name.contains("楼") || dist.name.contains("居") )){
nameFlag = true
}
}
def fieldFlag = false
if(dist.finish_building_date && dist.build_type){
fieldFlag = true
}
if(!tradeCount && !count && !fieldFlag){
def rowCount = findCount(dist._id.toString())
if(rowCount || nameFlag){
delIndex.addAndGet(1)
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',q))
println "[ ${delIndex.get()}/${totalCount.get()} ] "+city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
}else{
println city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
}
rets << [dist._id.toString(),dist.city, dist.region, dist.name, dist.address]
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\建行\\房产估值\\多余小区_${city}.xlsx",rets, "无房源")
}
def findCount(distId){
def sql = "select count(*) as count from fjw_community_artifi_check where commid='${distId}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size();
}
static def findHouseCount(def city, def region, def name, def alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
def aliasStr = escapeExprSpecialWord(alias.join("','"))
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and region = '"+region+"'")
sqlBuffer.append(" and district_name in ('${aliasStr}')")
sqlBuffer.append(" and (cat = 'rent' or cat='sell')")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def count = null
while (rs.next()){
count = rs.getInt("count")
}
return count;
}
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def gzRun(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\要删除小区 (5).xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
index++
println index
}
},0)
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
deleteAllRefDistrict(id)
}
},0)
}
def deleteAllRefDistrict(id){
def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"]
for(def tableName : tables){
deleteByCommid(tableName, id)
}
}
def deleteCommPrice(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\均价删除.xlsx",{d->
if(d['B']){
def id = d['B']
println id
deleteByCommid("community_base_price_gj",id)
}
},0)
}
def deleteByCommid(tableName, commid){
def sql = "delete from "+tableName+" where commid='${commid}'";
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
}
@@ -0,0 +1,85 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
@BatchOwner(Watcher.GaoJuan)
class DistrictGuidePriceBatch extends BaseBatch{
static main(args) {
runBatch(new DistrictGuidePriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findGuidePrice(city)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets)
println city + " " + rets.size()
}
}
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false);
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.match_region+"##"+ps.match_name)
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << areaData.district_code
ret << areaData.district
ret << ps.avg_price
ret << ps.source_type
ret << ""
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}else{
println ps.match_region+"##"+ps.match_name
}
}
}
return rets;
}
}
@@ -0,0 +1,79 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
/**
* 建行模型估值项目
* 估价系数数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class EvaluateRateBatch extends BaseBatch{
static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver")
static main(args) {
runBatch(new EvaluateRateBatch(), args);
}
@Override
public void process(Object args) {
def allDatas = createDatas(findEvaluationRate())
def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false);
}
def createDatas(def datas){
def allRows = []
datas.each { m ->
if(m.rank_remark!="开发商装修"){
def data = []
data << m.id.toString()
data << m.rate_type
data << m.rate
data << m.rank
data << "全国"
data << converFitment(m.rank_remark)
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
allRows << data
}
}
return allRows
}
private def converFitment(rank_remark){
if(rank_remark){
if(rank_remark.contains("简单装修")){
return "简装"
}else if(rank_remark.contains("精装修")){
return "精装"
}else if(rank_remark.contains("豪华装修")){
return "豪装"
}
}
return rank_remark
}
private def findEvaluationRate(){
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'";
def rets = []
sqlInstance.eachRow(sql,{
rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated]
});
return rets;
}
}
@@ -0,0 +1,819 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.sql.ResultSet
class ExportOracleData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new ExportOracleData()
task.run()
}
def run(){
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportCommunityStaging()
// exportCommunityBuileType()
// exportBigdataDistrictData()
// exportGuidedPrice()
// exportCommunityGuidedPrice()
// exportBigdataDeal()
exportThirdPartyPrice()
// exportBigdataCommunity()
// exportBigdataForsale()
// exportAvmEvaluationGJNB()
// exportAVMEvaluationItemGJNB()
// exportCommunityBasePrice()
// exportCommunityBasePriceGj()
// exportCityPrice()
// exportRegionPrice()
// exportLastFile()
}
def exportEvaluationRateData(){
def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.RATE_TYPE
data << r.RATE
data << r.RANK
data << r.CITY
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
def dataCodeFileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok"
}
def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "上传Ftp完成......"
}
def exportBigdataPoiData(){
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where cityId in (350200,150100)";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.ID
data << r.POI_NAME
data << r.TYPE
data << r.TYPE_CODE
data << r.LATITUDE
data << r.LONGITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok"
}
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "BIGDATA_POI_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "BIGDATA_POI_GJ 上传Ftp完成......"
}
def exportCommunityStaging(){
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.COMMID
data << r.MASTER_COMMID
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok"
}
def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "COMMUNITY_STAGING_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "COMMUNITY_STAGING_GJ 上传Ftp完成......"
}
def exportCommunityBuileType(){
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID in (350200,150100)";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok"
}
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
}
def exportBigdataDistrictData(){
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where cityId in (350200,150100)";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.LJXZQH
data << r.LJXZQHDM
data << r.XZQHDM
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok"
}
def headers = []//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "BIGDATA_DISTRICT_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "BIGDATA_DISTRICT_GJ 上传Ftp完成......"
}
def exportGuidedPrice(){
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate<TO_DATE('2021-09-01', 'yyyy-MM-dd') and cityname='上海'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.DEALDATE
data << r.AVG_PRICE
data << r.GUIDING_PERSON
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "GUIDED_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityGuidedPrice(){
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.AVGPRICE
data << r.SOURCETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202107")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......"
}
def exportBigdataDeal(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where cityId in (350200,150100) and dealdate>=to_date('2021-08-01','yyyy-mm-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.CJJE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "BIGDATA_DEAL_GJ 上传Ftp完成......"
}
def exportThirdPartyPrice(){
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" +
"TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" +
"from third_party_price_gj";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.ZHUGE_AVG_PRICE
data << r.ANJUKE_AVG_PRICE
data << r.FANG_AVG_PRICE
data << r.LIANJIA_AVG_PRICE
data << r.CITYHOUSE_AVG_PRICE
data << r.MAITIAN_AVG_PRICE
data << r.KE_AVG_PRICE
data << r.TAOFANG_AVG_PRICE
data << r.ZHONGYUAN_AVG_PRICE
data << r.TBS_AVG_PRICE
data << r.QFANG_AVG_PRICE
data << r.FANGTAN_AVG_PRICE
data << r.FANGDUODUO_AVG_PRICE
data << r.WIWJ_AVG_PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
// CcbFtpUtils.ftpUpload(fileName, "202107")
// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
// println "THIRD_PARTY_PRICE_GJ 上传Ftp完成......"
}
def exportBigdataCommunity(){
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100)
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.ok"
}
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......"
// CcbFtpUtils.ftpUpload(fileName, "202109")
// CcbFtpUtils.ftpUpload(dataCodeFileName, "202109")
// println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......"
}
def exportBigdataForsale(){
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where PUBLISHTIME>=to_date('2021-09-01','yyyy-mm-dd')";//(cityId=350200 or cityId=150100) and
def allRows = []
def index = 0
sqlInstance.eachRow(sql,{r->
index = index+1
println index
def data = []
data << r.CITYID
data << r.FORSALEID
data << r.PROPCODE
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.UNITPRICE
data << r.TOTALPRICE
data << r.PAYTYPE
data << r.FLOORNUM
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.HOUSESTRU
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEASPECT
data << r.FITMENT
data << r.IFELEVATOR
data << r.ELEVATORHOUSE
data << r.PUBLISHTIME
data << r.EXPIRETIME
data << r.IFFIRST
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.PUBLISHER
data << r.PHONE
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.ATTACH
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "BIGDATA_FORSALE_GJ 数据文件输出完成......"
// CcbFtpUtils.ftpUpload(fileName, "202107")
// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
// println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def exportAvmEvaluationGJNB(){
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where (cityId=350200 or cityId=150100) and year_month<202108 ";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMMUNITY_NUM
data << r.HAS_PRICE_COMMUNITY_NUM
data << r.PRICE_COVERAGE
data << r.ARTIFICIAL_CALC_NUM
data << r.ARTIFICIAL_CALC_COVERAGE
data << r.SAMPLE_CALC_NUM
data << r.SAMPLE_CALC_COVERAGE
data << r.ANN_CALC_NUM
data << r.ANN_CALC_COVERAGE
data << r.TEST_NUM
data << r.TEST_COMM_NUM
data << r.TEST_COMM_COVERAGE
data << r.PPE15_COVERAGE
data << r.RATE_MEDIAN
data << r.COEFFICIENT
data << r.DEAL_TEST_CASE_NUM
data << r.DEAL_TEST_COMM_NUM
data << r.DEAL_TEST_COMM_COVERAGE
data << r.DEAL_PPE15_COVERAGE
data << r.DEAL_COEFFICIENT
data << r.DEAL_RATE_MEDIAN
data << r.THIRD_TEST_CASE_NUM
data << r.THIRD_TEST_COMM_NUM
data << r.THIRD_TEST_COMM_COVERAGE
data << r.THIRD_PPE15_COVERAGE
data << r.THIRD_COEFFICIENT
data << r.THIRD_RATE_MEDIAN
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok"
}
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202107")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
}
def exportAVMEvaluationItemGJNB(){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where (cityId=350200 or cityId=150100) and year_month>=202108";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.CONFIDENCE
data << r.TARGET_AVG_PRICE
data << r.DEALDATE
data << r.UNIT_PRICE
data << r.BUILDAREA
data << r.FITMENT
data << r.HOUSEASPECT
data << r.FLOORNUM
data << r.FLOORCNT
data << r.DATA_TYPE
data << r.RATE
data << r.PE
data << r.CREATETIME
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
}
def exportCommunityBasePrice(){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where (cityId=350200 or cityId=150100) and year_month<202108";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202107")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityBasePriceGj(){
}
def exportCityPrice(){
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where (cityId=350200 or cityId=150100) and year_month>=202108";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "CITY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "CITY_PRICE_GJ 上传Ftp完成......"
}
def exportRegionPrice(){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where (cityId=350200 or cityId=150100) and year_month>=202108";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
}
def exportLastFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
def dataCodeFileName = "/ccb/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok"
}
CcbFtpUtils.ftpUpload(dataCodeFileName, "${curMonth}")
}
}
@@ -0,0 +1,211 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 人工指导价同步
*/
@BatchOwner(Watcher.GaoJuan)
class GuiDingPriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
static main(args) {
runBatch(new GuiDingPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets)
println city + " " + rets.size()
}
//添加搜房的参考价
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
// def datas = findFangDatas(districts)
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
// rows.addAll(fangRets)
//添加保亭文昌的指导价
def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
def cityDataMap = [:]
for(def d : cityDatas){
cityDataMap.put(d['A'], d['E'])
}
def personDatas = findPersonDatas(districts, cityDataMap);
def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
rows.addAll(personRets)
}
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
}
def findFangDatas(def districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println " 小区量:"+districts.size()
def result = Collections.synchronizedList(new ArrayList());
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findMatchThirdPrice(dist)
if(obj && obj.avg_price){
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def findPersonDatas(def districts, def dataMap){
def result = []
for(def i=0;i<districts.size();i++) {
def dist = districts[i];
def price = dataMap.get(dist._id.toString())
if (price) {
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
println dist.city + " " + dist.region + " " + dist.name + " " + price
}
}
return result;
}
def findMatchThirdPrice(dist){
def q = new BasicDBObject()
q.put("match_city", dist.city)
q.put("match_region", dist.region)
q.put("match_name", dist.name)
q.put("s_date", sDate)
q.put("source","搜房")
def obj = dataProcessColl.findOne(q)
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def createDatas(cityCodeDatas, distCodeMap, datas){
def rets = []
if(datas) {
for (def ps : datas) {
def ret = []
def areaData = distCodeMap.get(ps.region)
if (areaData) {
ret << ps.row_id
ret << ps.dist_id
ret << ps.name
ret << cityCodeDatas.get(ps.city)
ret << ps.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.avg_price
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
}
@@ -0,0 +1,98 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
static main(args) {
def task = new LoadGuide()
task.run()
}
def run(){
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
}
def savePrice(filePath){
def datas = ExcelUtils.read(filePath,0, 1)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['A']
o.name = d['C']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
println rets.size()
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
def saveDongGuangPrice(def filePath){
def datas = ExcelUtils.read(filePath,0, 0)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['A']
o.name = d['B']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.house_count = d['C'].toString().toDouble().toInteger()
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
def saveChengduoPrice(def filePath){
def datas = ExcelUtils.read(filePath,2, 0)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['B']
o.name = d['C']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class LocationUpdate {
static main(args) {
def task = new LocationUpdate();
task.run()
}
def run(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度\\经纬度人工核实结果_0923.xlsx",{d->
index++
if(d['C']){
println index +" "+d['A']+" "+d['B']+" "+d['C']
def region = RegionData.formatRegion(d['A'], d['B'])
if(d['H']){
def status = d['H'].toString().toDouble().toInteger()
if(status==1){
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble())
def lng2 = null
def lat2 = null
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
BasicDBObject q=new BasicDBObject()
q.put("m_date", new Date())
q.put("lat2", lat2)
q.put("lng2", lng2)
q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue())
q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}else{
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}
}else{
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}
}
})
}
}
@@ -0,0 +1,85 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 分期数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class StagingDistrictBatch extends BaseBatch{
static main(args) {
runBatch(new StagingDistrictBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap)
rows.addAll(rets)
println city +" 分期量:"+rets.size()
}
def results =[]
def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
results << headers
results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
}
def createCityData(def dists, def fenQiMap){
def rets = []
for(def d : dists){
if(d.master_district_name && d.master_district_name!="master"){
def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name)
if(masterDist){
def ret = []
ret << new ObjectId().toString()
ret << d._id.toString()
ret << masterDist._id.toString()
ret << ""
ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
// else{
// def udata = new BasicDBObject()
// udata.put("master_district_name", null)
// udata.put("m_date", new Date())
// println d.city +" "+d.name+" "+d._id.toString()
// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata))
// }
}
}
return rets;
}
def getFenQiMap(def dists){
def fenQiMap = [:]
for(def d : dists){
// if(d.master_district_name){
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
}
return fenQiMap;
}
}
@@ -0,0 +1,377 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.*;
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
//def citys = CcbCommonUtils.citys;
//def rows = []
//
//for(def city : citys) {
// def data = CcbCommonUtils.findCityCodeValue()
// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
//
// def count = 0
// sqlInstance.execute(sql,{
// count = it.count
// });
// println city +" "+count
//}
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
// dealMap.put(d['A'], d['B'])
//}
//
//
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){
// d['Q'] = dealMap.get(d['C'])
// }
//},2)
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
// def map = [:]
// map.put("land_area",d['D'])
// map.put("build_area",d['F'])
// map.put("greeningrate_area",d['H'])
// map.put("parkingcnt_area",d['J'])
// map.put("zts_area",d['L'])
// map.put("buildyear_area",d['N'])
// map.put("plotratio",d['P'])
//
// dealMap.put(d['A'], map)
//}
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//小区量
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def city = d['B']
// if (city) {
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['O'] = total
// }
//},0)
////小区字段覆盖率
//ExcelUtils.multiThreadingModify(fileName,{d->
// def city = d['A']
// if(city){
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['AA'] = total
//
// def a = findEstateType(cityid, sqlInstance)
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
//
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
//
// def c = findFiledCount(cityid, 'address', sqlInstance)
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
//
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
//
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
//
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
//
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
//
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
//
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
//
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
// println d
// }
//},1)
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx"
//def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['A']){
// d['U'] = dealMap.get(d['A'])
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx"
def datas = ExcelUtils.read(fileName, 0, 1)
def dealMap = [:]
for(def d : datas){
dealMap.put(d['A'], d['B'])
}
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d->
if(d['A']){
d['W'] = dealMap.get(d['A'])
}
},0)
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
//def updateIndex = 0
//def deleteIndex = 0
//ExcelUtils.modify(fileName,{d->
// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){
// def dist = CommonDatautils.findDistrictById(d['D'])
// def city = dist.city
// def region = RegionData.formatRegion(city, d['Q'])
// if(region && dist.region!=region){
// println city +" new:"+region +" old:"+d['C']
// def udata = new BasicDBObject()
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// deleteIndex++
// udata.put("status", 1)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }else{
// updateIndex++
// udata.put("region", region)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }
// }else{
// println d['D']+" 不存在"
// }
// }
//},0)
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-08-01','yyyy-mm-dd') and dealdate<to_date('2021-09-01','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
sqlInstance.eachRow(sql,{r->
map.put(r.cityname, r.count)
});
return map
}
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findFiledCount(cityid, field, sqlInstance){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
//def updateIndex = 0
//def delIndex = 0
//def cityMap = [:]
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(!d['R']){
//
// def obj = CommonDatautils.findDistrictById(d['D'])
// if(obj){
// def ranges = cityMap.get(obj.city)
// if(!ranges){
// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
// ranges = []
// for(def r : dbcursor){
// ranges << r
// }
// cityMap.put(obj.city, ranges)
// }
//
// if(ranges){
// def lng = obj.lng/10000000000.0
// def lat = obj.lat/10000000000.0
//
// def region = null
// for(def r : ranges){
// try{
// def flag = isInArea(lng, lat, r.polyline)
// if(flag){
// def regionStr = r.region
// region = RegionData.formatRegion(obj.city, regionStr)
// break
// }
// }catch(Exception e){
// println e.getMessage()
// }
//
// }
// if(region && region!=obj.region){
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
// }else if(region && region==obj.region){
// println "[same] "+obj.city +" "+obj.name
// }else{
// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
// delIndex++
// println delIndex +" [delete] "+obj.city +" "+obj.name
// }
// }else{
// println obj.city +" 没有此城市边框信息"
// }
//
// }else{
// println d['D']+" 没找到该小区"
// }
//
// }
//},0);
//def city = "杭州"
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city))
//def ranges = []
//for(def d : dbcursor){
// ranges << d
//}
//def index =0
//def updateIndex = 0
//def delIndex = 0
//for(def dist : dists){
//// if(dist.region=="江干"){
// def lat = dist.lat/10000000000.0
// def lng = dist.lng/10000000000.0
//
// def regionStr =null
// def region = null
// for(def r : ranges){
// def flag = isInArea(lng, lat, r.polyline)
// if(flag){
// regionStr = r.region
// region = RegionData.formatRegion(dist.city, regionStr)
// break
// }
// }
// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
// if(regionStr && region!=dist.region){
//
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// delIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
// println delIndex+" [del]"+region +" "+dist.name
// }else{
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name
// }
// }else{
// index++
// println index+" [没找到]"+dist.name
// }
//// }
//}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
//Point point = new Point(lng, lat);
//def flag = InsidePolygon.aboutZone(point, pointList)
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
def getStreetInfo(def city, def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent && addressComponent.city.contains(city)){
township = addressComponent.township
}
}
}
return township;
}
@@ -0,0 +1,81 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class TestRepeat {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new TestRepeat()
task.run()
}
def run(){
}
def checkUpdateCount(){
def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat")
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(newHouses.size())
println " ·¿Ô´Á¿£º"+newHouses.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<newHouses.size();i++){
final def map = newHouses[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findHouse(map.houseId, map.commId)
if(obj){
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findHouse(def houseId, def commId){
def sql = "select * from bigdata_forsale_gj where commid='${commId}' and forsaleid='${houseId}'";
def obj = null
sqlInstance.eachRow(sql,{r->
obj = r
});
return obj
}
def readTxt(def file){
def lines = org.apache.commons.io.FileUtils.readLines(new File(file))
def rets = [];
for(line in lines) {
def arrs = line.split("|@|")
def map = [:]
map.houseId = arrs[1]
map.commId = arrs[6]
map.pdate = arrs[10]
rets << map
}
return rets;
}
}
@@ -0,0 +1,294 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
/**
* 建行模型估值项目
* 第三方均价数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202108//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202106
def endDate = 202108
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThirdPartyPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
for(def city : citys){
index++
def statMap = getDistrictStatByCity(city, weekend)
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city, statMap)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
rows.addAll(rets)
println index+" "+city+" "+rets.size()
}
def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat"
}
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
}
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def rets = []
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
if(data){
def ret = []
ret << third.id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.get(d.city)
ret << d.city
ret << data.district_code
ret << data.district
ret << sDate
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
}else{
ret << ""
}
if(third.anjuke_esf_price){
ret << third.anjuke_esf_price
}else{
ret << ""
}
if(third.soufang_esf_price){
ret << third.soufang_esf_price
}else{
ret << ""
}
if(third.lianjia_esf_price){
ret << third.lianjia_esf_price
}else{
ret << ""
}
if(third.cityhouse_esf_price){
ret << third.cityhouse_esf_price
}else{
ret << ""
}
if(third.maitian_esf_price){
ret << third.maitian_esf_price
}else{
ret << ""
}
if(third.beike_esf_price){
ret << third.beike_esf_price
}else{
ret << ""
}
if(third.get("365house_esf_price")){
ret << third.get("365house_esf_price")
}else{
ret << ""
}
if(third.zhongyuan_esf_price){
ret << third.zhongyuan_esf_price
}else{
ret << ""
}
if(third.tuboshi_esf_price){
ret << third.tuboshi_esf_price
}else{
ret << ""
}
if(third.qfang_esf_price){
ret << third.qfang_esf_price
}else{
ret << ""
}
if(third.fangtan_esf_price){
ret << third.fangtan_esf_price
}else{
ret << ""
}
if(third.fangduoduo_esf_price){
ret << third.fangduoduo_esf_price
}else{
ret << ""
}
if(third.get("5i5j_esf_price")){
ret << third.get("5i5j_esf_price")
}else{
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
dataMap.put("id", db._id.toString())
map.put(key, dataMap)
}
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
dataMap.put("qfang_esf_price", db.qfang_esf_price)
}
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
}
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
}
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
def startPrice = (int)(price - price*0.3)
def endPrice = (int)(price*0.3 + price)
value.each{pk,pv->
if(pk.contains("price")){
if(pv>=startPrice && pv<=endPrice){
priceMap.put(pk,pv)
}
}else{
priceMap.put(pk,pv)
}
}
if(priceMap.size()>1){
resultMap.put(key, priceMap)
}
}
}else{
resultMap.put(key, value)
}
}
}
return resultMap
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
return map
}
}
@@ -0,0 +1,652 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/***
* 建行成交抓取统计
*/
class TradeStatLog {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new TradeStatLog()
// task.run()
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
task.exportTrade()
// task.deleteDistrict()
}
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-06-01"
def date = "2021-07-01"
def endDate = "2021-08-01"
def newDate = "2021-09-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def nearMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
def ret = []
ret << city
if(city == "上海"){
def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def nearMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger()
def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger()
def c = nearMonthCount.toInteger() + nearMonthYoudaCount.toInteger()
ret << a
ret << b
ret << c
def d = (c-b)*100/a
def pre = DataHandleUtils.getDoubleRound(2, d)
ret << pre
}else{
ret << lastMonthCount
ret << curMonthCount
ret << nearMonthCount
if(curMonthCount){
def a = (nearMonthCount-curMonthCount)*100/curMonthCount
def pre = DataHandleUtils.getDoubleRound(2, a)
ret << pre
}else{
ret << null
}
}
for(def source : sources){
def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def sourceNearMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
ret << sourceLastMonthCount
ret << sourceCurMonthCount
ret << sourceNearMonthCount
}
rets << ret
println ret
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
def thirdPartyStat(){
def citys = CcbCommonUtils.citys;
def rets = []
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"]
for(def city : citys){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat","lng"]);
def m = getThirdPartyPricePrecent(city)
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
println rets.get(rets.size()-1)
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计0927.xlsx", rets, "数据")
println "表格生成完成"
}
public static def findTradesCount(city, source, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
if(source){
q.put("source", source)
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def count = DEAL_PRICE_COLL.count(q)
return count
}
def getYouDaData(startDate, endDate){
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lt', endDate));
def count = youda_esfhouse_coll.count(q)
return count;
}
def getThirdPartyPricePrecent(def city){
def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108";
def count0 = 0
def count1 = 0
def count2 = 0
def count3 = 0
def count4 = 0
def count5 = 0
def count6 = 0
def count7 = 0
def count8 = 0
def count9 = 0
def count10 = 0
def count11 = 0
def count12 = 0
def count13 = 0
def count14 = 0
def total = 0
sqlInstance.eachRow(sql,{third->
total++
def ret = []
if(third.zhuge_avg_price){
ret << third.zhuge_avg_price
}
if(third.anjuke_avg_price){
ret << third.anjuke_avg_price
}
if(third.fang_avg_price){
ret << third.fang_avg_price
}
if(third.lianjia_avg_price){
ret << third.lianjia_avg_price
}
if(third.cityhouse_avg_price){
ret << third.cityhouse_avg_price
}
if(third.maitian_avg_price){
ret << third.maitian_avg_price
}
if(third.ke_avg_price){
ret << third.ke_avg_price
}
if(third.taofang_avg_price){
ret << third.taofang_avg_price
}
if(third.zhongyuan_avg_price){
ret << third.zhongyuan_avg_price
}
if(third.tbs_avg_price){
ret << third.tbs_avg_price
}
if(third.qfang_avg_price){
ret << third.qfang_avg_price
}
if(third.fangtan_avg_price){
ret << third.fangtan_avg_price
}
if(third.fangduoduo_avg_price){
ret << third.fangduoduo_avg_price
}
if(third.wiwj_avg_price){
ret << third.wiwj_avg_price
}
if(ret.size()==0){
count0++
}
if(ret.size()==1){
count1++
}
if(ret.size()==2){
count2++
}
if(ret.size()==3){
count3++
}
if(ret.size()==4){
count4++
}
if(ret.size()==5){
count5++
}
if(ret.size()==6){
count6++
}
if(ret.size()==7){
count7++
}
if(ret.size()==8){
count8++
}
if(ret.size()==9){
count9++
}
if(ret.size()==10){
count10++
}
if(ret.size()==11){
count11++
}
if(ret.size()==12){
count12++
}
if(ret.size()==13){
count13++
}
if(ret.size()==14){
count14++
}
});
def pre1 = 0
def pre2 = 0
def pre3 = 0
def pre4 = 0
def pre5 = 0
def pre6 = 0
def pre7 = 0
def pre8 = 0
def pre9 = 0
def pre10 = 0
def pre11 = 0
def pre12 = 0
def pre13 = 0
def pre14 = 0
if(total){
pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total)
pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total)
pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total)
pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total)
pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total)
pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total)
pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total)
pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total)
pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total)
pre10 = DataHandleUtils.getDoubleRound(2, count10*100.0/total)
pre11 = DataHandleUtils.getDoubleRound(2, count11*100.0/total)
pre12 = DataHandleUtils.getDoubleRound(2, count12*100.0/total)
pre13 = DataHandleUtils.getDoubleRound(2, count13*100.0/total)
pre14 = DataHandleUtils.getDoubleRound(2, count14*100.0/total)
}
def map = [:]
map.total = total
map.count1 = pre1
map.count2 = pre2
map.count3 = pre3
map.count4 = pre4
map.count5 = pre5
map.count6 = pre6
map.count7 = pre7
map.count8 = pre8
map.count9 = pre9
map.count10 = pre10
map.count11 = pre11
map.count12 = pre12
map.count13 = pre13
map.count14 = pre14
return map
}
def averageStat(){
def citys = ["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
def map = [:]
for(def city : citys){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
def rets = findThirdPriceByDist(districts)
map.put(city, rets.size())
map.put(city+"-count", districts.size())
println districts.size()+" "+ rets.size()
}
println map
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\估值测试数据覆盖率占比.xlsx",{d->
def city = d['A']
def count = map.get(city)
def distCount = map.get(city+"-count")
d['H'] = count
d['J'] = distCount
def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount)
d['I'] = pre
},0)
}
def findThirdPriceByDist(districts){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def count = findAverageStat(dist)
if(count){
rets << dist
println totalCount.get() +"/${districts.size()} "+dist.region+" "+dist.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def findAverageStat(dist){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("name", dist.name)
q.put("category","district")
q.put("s_date", new BasicDBObject('$gte', 202105).append('$lt',202108))
def dbcursor = AVERAGE_STATISTICS.find(q)
def flag = 0
for(def d : dbcursor){
if(!flag){
if(d.zhugezhaofang_esf_price){
flag = 1
continue
}
if(d.anjuke_esf_price){
flag = 1
continue
}
if(d.get("58tc_esf_price")){
flag = 1
continue
}
if(d.soufang_esf_price){
flag = 1
}
if(d.lianjia_esf_price){
flag = 1
continue
}
if(d.cityhouse_esf_price){
continue
flag = 1
continue
}
if(d.maitian_esf_price){
flag = 1
continue
}
if(d.beike_esf_price){
flag = 1
continue
}
if(d.get("365house_esf_price")){
flag = 1
continue
}
if(d.zhongyuan_esf_price){
flag = 1
continue
}
if(d.tuboshi_esf_price){
flag = 1
continue
}
if(d.qfang_esf_price){
flag = 1
continue
}
if(d.fangtan_esf_price){
flag = 1
continue
}
if(d.fangduoduo_esf_price){
flag = 1
continue
}
if(d.get("5i5j_esf_price")){
flag = 1
continue
}
}
}
return flag
}
def exportTrade(){
def citys = CcbCommonUtils.citys;
// def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-07-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-08-01","yyyy-MM-dd"))
// def trades = getYouDaData()
//def statMap = getDistrictStatByCity(city, weekend)
def nameMap = getAllDistrict(city)
def rMap = findDealIds(city);
def ids = rMap.ids
def details = rMap.detail
for(def t : trades){
if(!ids.contains(t._id.toString())){
def distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name)
if(!distId && (t.district_name.toString().endsWith("弄") || t.district_name.toString().endsWith("号"))){
distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name+"小区")
}
if(!distId){
def flag = t.district_name.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
def datas = CommonDatautils.findDistrictByAddress2(t.city, t.region, t.district_name, "district")
if(datas && datas.size()==1){
distId = datas.get(0)._id.toString()
}
}
}
def reason = ""
if(distId){
def dist = CommonDatautils.findDistrictById(distId)
// def price = statMap.get(dist.region+"@"+dist.name)
// def startPrice = null
// def endPrice = null
// if(price){
// if(dist.city=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else if(dist.city=="杭州"){
// startPrice = (int)(price - price*0.15)
// endPrice = (int)(price*0.15 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// }
// if(t.avg_price<startPrice || t.avg_price>endPrice){
// reason = "价格过滤"
// }
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
reason = "小区过滤"
}
if(!reason){
def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
if(details.contains(key)){
reason = "数据重复"
}
}
}
def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
//rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type]
rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason]
}
}
println city +" "+rets.size()
}
ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据")
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
deleteAllRefDistrict(id)
}
},0)
}
def deleteAllRefDistrict(id){
def tables = ["bigdata_community_gj"]
for(def tableName : tables){
deleteByCommid(tableName, id)
}
}
def deleteByCommid(tableName, commid){
def sql = "delete from "+tableName+" where commid='${commid}'";
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
def getYouDaData() {
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))
def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd")
def q = new BasicDBObject()
// q.put("premisesName","共富一村")
q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def trades = []
for(def db : dbcursor){
def map = [:]
map._id=db._id.toString()
map.city = "上海"
map.region = db.region
if(db.districtName && db.districtName!="null"){
map.district_name=db.districtName
}else if(db.address){
map.district_name=db.address
}
map.area = db.area
map.trade_date = db.contractTime
map.avg_price= db.price
map.estate_type = db.houseType
trades << map
}
return trades
}
def findDealIds(city){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate<to_date('2021-08-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
def region = RegionData.formatRegion(r.cityname, r.distname)
def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd")
if(!detailInfos.contains(key)){
detailInfos << key
}
}
});
def rMap = [:]
rMap.ids = ids
rMap.detail = detailInfos
return rMap
}
public static def findTradesDetail(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
// q.put("district_name","百合花园")
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
def getAllDistrict(city){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
def map = [:]
for(def d : dists){
map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString())
def alias = d.alias
if(alias){
for(def name : alias){
map.put(d.city+"##"+d.region+"##"+name, d._id.toString())
}
}
}
return map
}
}
@@ -0,0 +1,392 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class UpdateHouseRegion {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new UpdateHouseRegion()
// task.updateSeawaterRegion()
// task.updateDealRegion()
// task.updateThirdPartyRegion()
task.updateNoMatchDealRegion()
// task.addAlias()
// task.addFenQi()
}
def addFenQi(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
addFenQiAlias(dists)
println city +" 处理完成"
}
}
def addFenQiAlias(dists){
for(def dist : dists){
def s1 = DataUtil.match(dist.name, Pattern.compile("[一|二|三|四|五|六|七|八|九|十|十一|十二|十三|十四|十五|十六|十七|十八|\\w+](期|区)\$"))
if(s1){
def newName = dist.name.replace("Ⅰ","1").replace("Ⅱ","2").replace("A区","a区").replace("B区","b区").replace("C区","c区").replace("D区","d区").replace("E区","e区").replace("F区","f区").replace("一期","1期").replace("二期","2期").replace("三期","3期").replace("四期","4期").replace("五期","5期").replace("六期","6期").replace("七期","7期").replace("八期","8期").replace("九期","9期").replace("十期","10期").replace("十一期","11期").replace("十二期","12期")
if(newName!=dist.name){
def alias = dist.alias
if(!alias){
alias = []
}
if(!alias.contains(newName)){
alias << newName
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+alias
}
}
}
}
}
def addAlias(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
addDistricts(dists)
println city +" 处理完成"
}
}
def addDistricts(dists){
for(def dist : dists){
def alias = dist.alias
if(alias){
def oldCount = alias.size()
def newAlias = []
for(def aliasName : alias){
if(aliasName.contains("·") || aliasName.contains(".")){
def newName = aliasName.replace("·","").replace(".","")
newAlias << aliasName
if(!newAlias.contains(newName)){
if(newName!=dist.name){
newAlias << newName
}
}
}else{
if(!newAlias.contains(aliasName)) {
newAlias << aliasName
}
}
}
def newCount = newAlias.size()
if(oldCount && alias!=newAlias){
def udata = new BasicDBObject()
udata.put("alias", newAlias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+newAlias
}
}
}
}
def updateThirdPartyRegion(){
def city = "东莞"
def rets = findThirdPartyPrice(city)
batchThirdPartyUpdate(city, rets);
}
def findThirdPartyPrice(city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date",new BasicDBObject('$gte', 202101))
def dbcursor = thirdPartyColl.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findDealData(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01")));
def cursor = DEAL_PRICE_COLL.find(q).batchSize(2000)
def datas = []
def index = 0
for(def obj : cursor){
index++
datas << obj
println index
}
return datas
}
def updateDealRegion(){
def city = "东莞"
def datas = findDealData(city)
batchDealRegion(city, datas)
}
def batchDealRegion(def city, def datas){
def updateCount = new AtomicLong()
def curCount = new AtomicLong()
def totalCount = datas.size();
final CountDownLatch latch = new CountDownLatch(totalCount)
println city +" 成交总量:"+totalCount
for(def i=0;i<totalCount;i++) {
final def data = datas[i];
fixedThreadPool.execute(new Runnable() {
void run() {
curCount.addAndGet(1)
try {
def region = data.region
def name = data.district_name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
}
def updateNoMatchDealRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d->
if(!d['J']){
def city = d['B']
def region = d['C']
def name = d['D']
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata))
d['J'] = "原数据区域修改("+dist.region+")"
println d['A']+" "+region +" "+dist.region
}
}
}
},0)
}
def updateSeawaterRegion(){
def city = "东莞"
def rets = findHouseSource(city, 20190701, 20211001)
batchUpdate(city, rets);
}
def batchThirdPartyUpdate(def city, def thirdDatas){
def updateCount = new AtomicLong()
def curCount = new AtomicLong()
def totalCount = thirdDatas.size();
final CountDownLatch latch = new CountDownLatch(totalCount)
println city +" 第三方总量:"+totalCount
for(def i=0;i<totalCount;i++) {
final def data = thirdDatas[i];
fixedThreadPool.execute(new Runnable() {
void run() {
curCount.addAndGet(1)
try {
def region = data.region
def name = data.name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("region", dist.region)
thirdPartyColl.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
}
def findCount(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate)
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
while (rs.next()) {
def count = rs.getInt("count")
println count
}
}
def batchUpdate(def city, def houses){
def updateCount = new AtomicLong()
def curCount = new AtomicLong()
def totalCount = houses.size();
final CountDownLatch latch = new CountDownLatch(totalCount)
println city +" 房源总量:"+totalCount
for(def i=0;i<totalCount;i++) {
final def house = houses[i];
fixedThreadPool.execute(new Runnable() {
void run() {
curCount.addAndGet(1)
try {
def region = house.region
def name = house.name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateCount.addAndGet(1)
updateHouseRegion2(house.url, dist.region);
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
}
def findHouseSource(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM \"seawater-2019\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def houses = []
while (rs.next()){
houses << ["region":rs.getString("region"), "name":rs.getString("district_name"),"url":rs.getString("url")]
}
return houses;
}
def updateHouseRegion2(def url, def region){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.206:9200/seawater-2019/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.region='${region}'",
"lang": "painless"
},
"query": {
"term": {
"url": "${url}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
// println(response)
}
def updateHouseRegion(def url, def region){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("update seawater-2021")
sqlBuffer.append(" set region = '"+region+"'")
sqlBuffer.append(" where url='"+url+"'")
def result = dataSource.connection.createStatement().executeUpdate(sqlBuffer.toString())
println region +" "+result
}
def getAllDistricts(def city){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"])
def map = [:]
for(def d : dists){
map.put()
}
}
}
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","东寓新村")
// q.put("name","金辉伯爵山")
DBObject keys = new BasicDBObject();
if(fields){
@@ -536,8 +536,8 @@ class CommonDatautils {
if(name){
alias << name;
}
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("region", region);
@@ -698,6 +698,28 @@ class CommonDatautils {
return rets;
}
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", new BasicDBObject('$in', regions))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllDistrictByQuery(def q, def cat, def fields){
@@ -758,6 +780,9 @@ class CommonDatautils {
keys.put("address", 1);
keys.put("build_structure", 1);
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("name_cat", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -865,6 +890,25 @@ class CommonDatautils {
}
return null;
}
public static def findDistrictsByName(String city, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
q.put("visible", 0);
def rets = []
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
rets << obj;
}
}
return rets;
}
public static def getSimpleDistrictMap(def cat, def city){
DBObject q = new BasicDBObject();
q.put("visible", 0);
@@ -1187,6 +1231,28 @@ class CommonDatautils {
return ret;
}
public static def findDistrictByAddress2(def city, def region, def address, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('visible',0)
q.put("address", address);
def dsCursor = getDataStoreConnection().find(q);
def ret = [];
for(def obj : dsCursor){
ret << obj;
}
return ret;
}
public static def getCityRegions(city){
@@ -1292,7 +1358,7 @@ class CommonDatautils {
}
public static def getAllDictionary(){
def dbCursor = DICT_COLL.find(new BasicDBObject())
def dbCursor = DICT_COLL.find(new BasicDBObject()).batchSize(5000)
def rets = []
def filters = [ "estate","hfword","main","metro","station"]
for(def obj : dbCursor){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202106.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -135,7 +135,7 @@ rets << ["9
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
println s
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
DingDingMessageUtil.sendTextMessage(access_token, s)
@@ -1,11 +1,16 @@
package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
@@ -63,7 +68,7 @@ class AddThirdPriceDistrict {
def resAddrMap = [:]
def filterEstateTypeMap = [:]
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -74,20 +79,84 @@ class AddThirdPriceDistrict {
static main(args) {
def task = new AddThirdPriceDistrict();
// task.run()
task.run()
def mapName = "锦华星光苑3,4幢"
// def mapName = "锦华星光苑3,4幢"
//// println task.pattern("港新花苑A1区");
println mapName.replaceAll("\\w+-?\\w+(幢)", "")
println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
// println mapName.replaceAll("\\w+-?\\w+(幢)", "")
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
}
def run(){
// exportDatas();
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// addInfo();
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// getDataType("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx")
// matchDistDatas("D:\\数据\\小区\\南昌第三方均价.xlsx")
addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
}
def dataFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
def name = row['C']
if(name){
def flag = NameProcess.filterName(name)
if(!flag){
row['T'] = 1
}else{
if(name.toString().length()<3){
if(row['N'] && row['N']>=10 && !row['J']){
row['T'] = 25
}else{
row['T'] = 18
}
}else{
def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){
row['T'] = 19
}
}
}
}else{
if(!name){
row['T'] = 1
}
}
});
println "3.名称过滤完成"
}
def addNewMapInfo(def path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['I']){
def gdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "高德")
if(gdData){
row['F'] = gdData.district
row['G'] = gdData.name
row['H'] = gdData.address
row['I'] = gdData.type
row['J'] = gdData.score
row['K'] = gdData.lng
row['L'] = gdData.lat
}
def bdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "百度")
if(bdData){
row['M'] = bdData.district
row['N'] = bdData.name
row['O'] = bdData.address
row['P'] = bdData.type
row['Q'] = bdData.score
row['R'] = bdData.lng
row['S'] = bdData.lat
}
}
},0);
}
def addCrawlMapInfo( def path){
@@ -105,27 +174,27 @@ class AddThirdPriceDistrict {
row['K'] = data.lat
}
def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
def bdRegion =null
def bdName = null
def bdAddress = null
def bdLng = null
def bdLat = null
def bdEstateType = null
if(baiduRets){
bdRegion = baiduRets.region
bdName = baiduRets.name
bdAddress = baiduRets.address
bdLng = baiduRets.lng
bdLat = baiduRets.lat
bdEstateType = baiduRets.estateType
}
row['L'] = bdRegion
row['M'] = bdName
row['N'] = bdAddress
row['O'] = bdEstateType
row['P'] = bdLng
row['Q'] = bdLat
// def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
// def bdRegion =null
// def bdName = null
// def bdAddress = null
// def bdLng = null
// def bdLat = null
// def bdEstateType = null
// if(baiduRets){
// bdRegion = baiduRets.region
// bdName = baiduRets.name
// bdAddress = baiduRets.address
// bdLng = baiduRets.lng
// bdLat = baiduRets.lat
// bdEstateType = baiduRets.estateType
// }
// row['L'] = bdRegion
// row['M'] = bdName
// row['N'] = bdAddress
// row['O'] = bdEstateType
// row['P'] = bdLng
// row['Q'] = bdLat
}
},0);
@@ -135,29 +204,32 @@ class AddThirdPriceDistrict {
def exportDatas(){
def city = "无锡"
def datas = findMatchErrorDatas(city);
println "共【"+datas.size()+"】条"
datas = filterName(datas);
println "过滤后共【"+datas.size()+"】条"
def resMap = groupByName(datas);
def rets = []
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
resMap.each {k,v->
def ret = []
def arrs = k.split("##")
ret << arrs[0]
ret << arrs[1]
if(arrs.size()==3 && arrs[2]){
ret << arrs[2]
}else{
ret << ""
//跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞",
def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
for(def city : citys){
def datas = findMatchErrorDatas(city);
println "共【"+datas.size()+"】条"
datas = filterName(datas);
println "过滤后共【"+datas.size()+"】条"
def resMap = groupByName(datas);
def rets = []
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
resMap.each {k,v->
def ret = []
def arrs = k.split("##")
ret << arrs[0]
ret << arrs[1]
if(arrs.size()==3 && arrs[2]){
ret << arrs[2]
}else{
ret << ""
}
ret << resAddrMap.get(k)
ret << v.join(",")
rets << ret
}
ret << resAddrMap.get(k)
ret << v.join(",")
rets << ret
ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后")
}
ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后")
}
def filterName(datas){
@@ -465,7 +537,7 @@ class AddThirdPriceDistrict {
def findMatchErrorDatas(def city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date",202005)
q.put("s_date",new BasicDBObject('$gte', 202101))
// q.put("name","博济聚创科技园")
// q.put("source","贝壳找房")
def dbcursor = matchErrorLogColl.find(q)
@@ -499,8 +571,8 @@ class AddThirdPriceDistrict {
return thirdPartyColl.findOne(q)
}
def addInfo(){
def addInfo(filePath){
province = "江西"
// def city = "苏州"
// def datas = findMatchErrorDatas(city);
// println "共【"+datas.size()+"】条"
@@ -509,8 +581,8 @@ class AddThirdPriceDistrict {
// def resMap = groupByName(datas);
def index = 0;
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx", { row ->
def flag = row['R']
ExcelUtils.multiThreadingModify(filePath, { row ->
def flag = row['T']
def dgDatas = row['G']
// if(!flag){
index++
@@ -544,9 +616,9 @@ class AddThirdPriceDistrict {
// def estateType = filterEstateTypeMap.get(row['A']+"##"+row['B']+"##"+row['C'])
// row['W'] = estateType
if(!row['X']){
def name = nameProcess(row['B'], row['C'])
row['X'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C'])
if(!row['Z']){
//def name = nameProcess(row['B'], row['C'])
row['Z'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C'])
}
//
@@ -953,11 +1025,11 @@ class AddThirdPriceDistrict {
def queryPoiMap(def city, def region, def name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and adname = '" + region + "'")
sqlBuffer.append(" and name = '" + name + "'")
sqlBuffer.append(" and source = '高德'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and source = '高德' order by score() desc limit 1")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def map = [:]
while (rs.next()) {
@@ -966,8 +1038,11 @@ class AddThirdPriceDistrict {
map.name = rs.getString("name")
map.address = rs.getString("address")
map.type = rs.getString("type")
map.lng = rs.getString("location.lon")
map.lat = rs.getString("location.lat")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
}
return map;
}
@@ -982,6 +1057,37 @@ class AddThirdPriceDistrict {
return null
}
def matchDistDatas(def path){
def index = 0
ExcelUtils.multiThreadingModify(path,{ List row ->
index ++
if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['C'], row['D']);
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
//row['J'] = rMap.city
row['U'] = rMap.region
row['V'] = rMap.name
row['W'] = rMap.address
if(rMap.lat){
// def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
// def maplng = baidu.x
// def maplat = baidu.y
// row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['C'], row['V'])
if(score<0.3){
score = CosineSimilarAlgorithm.getSimilarity(row['D'], row['V'])
}
row['X'] = score
}
}
println index +" "+row
}
},0)
println "5.小区匹配完成"
}
}
@@ -7,7 +7,7 @@ import java.sql.ResultSet
class EsSearchUtils {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -64,24 +64,29 @@ class EsSearchUtils {
sqlBuffer.append(" and region = '"+region+"'")
sqlBuffer.append(" and district_name = '"+name+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" limit 100")
sqlBuffer.append(" limit 200")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def rets = []
while (rs.next()){
def height = rs.getInt("height")
def direction = rs.getString("direction")
def fitment = rs.getString("fitment")
def room = rs.getInt("room")
def hall = rs.getInt("hall")
def toilet = rs.getInt("toilet")
def map = [:]
map.height = height
map.direction = direction
map.fitment = fitment
map.room = room
map.hall = hall
map.toilet = toilet
rets << map
if(room){
def height = rs.getInt("height")
def direction = rs.getString("direction")
def fitment = rs.getString("fitment")
def hall = rs.getInt("hall")
def toilet = rs.getInt("toilet")
def area = rs.getInt("area")
def map = [:]
map.height = height
map.direction = direction
map.fitment = fitment
map.room = room
map.hall = hall
map.toilet = toilet
map.area = area
rets << map
}
}
return rets;
}
@@ -259,6 +259,31 @@ class FiledStrand {
return updateFile;
}
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=1000 && v<=10000000){
return v.toString()
}
}
return null
}
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if(v>=1 && v<=100000){
try {
return v.toString().toInteger();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def addBuildAreaToData(def dist, def updateFile){
def build_area = null;
if(dist.build_area){
@@ -298,6 +323,26 @@ class FiledStrand {
}
return updateFile;
}
public static def processCubagePercent(cubage_percent, estate_type2){
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
if(estate_type2 == "别墅"){
if(v>=0.3 && v<=1){
return v.toString()
}
}else{
if(v>1 && v<=10){
return v.toString()
}
}
}
}
return null
}
static def addVirescencePercentToData(dist, updateFile){
def virescence_percent = null
if(dist.virescence_percent){
@@ -312,6 +357,16 @@ class FiledStrand {
return updateFile;
}
static def processVirescencePercent(virescence_percent){
if(virescence_percent){
def v = getValue(virescence_percent)
if(v && v>15 && v<=70){
return v.toString()
}
}
return null;
}
static def addBuildTypeToData(dist, updateFile){
if(dist.estateType2 && dist.buildType){
def house = ["超高层","小高层","多层","高层","花园洋房","老洋房","四合院"]
@@ -394,6 +449,7 @@ class FiledStrand {
}
return estateCharege;
}
static def processHouseholdCount(def household_count, def estateType2){
if(household_count){
household_count = household_count.toInteger()
@@ -57,8 +57,12 @@ class GaoDeMapCrawl extends BaseBatch{
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
def prefix = "company_"
def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
def prefix = "food_"
def categorys = ["050000"]
// def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
def totalCount = new AtomicLong()
static main(args) {
@@ -116,20 +120,25 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
// def citys = CommonUtils.findAllCity()
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = ["南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def allcitys = ["上海"]//CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def stop = false
while(stop == false){
def cityLations = findAllLation(city);
if(!cityLations){
stop = true;
}else{
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
for(def city : allcitys){
if(!citys.contains(city)){
def stop = false
while(stop == false){
def cityLations = findAllLation(city);
if(!cityLations){
stop = true;
}else{
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
}
}
}
}
}
@@ -141,7 +150,11 @@ class GaoDeMapCrawl extends BaseBatch{
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
// q.put("status", "失败")
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def results = []
for(def d : dbcursor){
results << d
}
return results;
}
@@ -182,14 +195,15 @@ class GaoDeMapCrawl extends BaseBatch{
}
def searching(i, location){
// def bound_gd = location.bound_gd
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]]
// lats.sort()
// lngs.sort()
// println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(i, location)
println i+" "+ location
// 120.019556,30.313701
//if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){
search(i, location)
// }
if(updateLists.size() > 0 && updateLists.size()>1000){
@@ -29,6 +29,8 @@ class OpenApiUtils {
rMap.region = dto.district
rMap.name = dto.name
rMap.id = dto.id
rMap.lng = dto.lng
rMap.lat = dto.lat
if(dto.address && dto.address!="null"){
rMap.address = dto.address
}
@@ -45,18 +45,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\农行\\肇庆农行支行房价通账号.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252580 + i;
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)";
def id = 252608 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王辰岑')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'肇庆','肇庆:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
onLineSqlInstance.execute(sql3)
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils
@@ -28,8 +29,9 @@ class RegionPointCrawl {
static main(args) {
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
@@ -55,7 +57,13 @@ class RegionPointCrawl {
}
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
if(ret.size()>0){
saveData(cityMap.name, m.name, m.ad_code , ret);
def standardCity = RegionData.formatCity(cityMap.name)
if(standardCity){
saveData(standardCity, m.name, m.ad_code , ret);
}else{
println cityMap.name +" 没有标准化"
}
}
}
}
@@ -72,7 +80,7 @@ class RegionPointCrawl {
for(def cityMap : cityList){
if(cityMap.level == "city"){
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
saveGaoDeRegionCode(province, cityMap.districts);
saveGaoDeRegionCode(province, cityMap.name, cityMap.districts);
}
}
}
@@ -81,10 +89,10 @@ class RegionPointCrawl {
}
}
def saveGaoDeRegionCode(province, districts){
def saveGaoDeRegionCode(province, cityName,districts){
if(districts){
for(def dist : districts){
if(dist.level=="district"){
if(dist.level=="district" || cityName=="东莞市"){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
@@ -221,4 +229,3 @@ class RegionPointCrawl {
return KeyList.get(count);
}
}
H
@@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
class WuXiAddressTest {
static main(args) {
def task = new WuXiAddressTest()
task.run()
//task.run()
task.addAddress()
}
def addAddress(){
ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d->
def prov = d['C']
def city = d['D']
def region = d['E']
def address = d['B']
def road = d['J']
def street = d['R']
def newAddress = prov+"省"+city +"市"
if(region == "惠山"){
newAddress = newAddress + region +"区"
}else{
newAddress = newAddress + region +"市"
}
d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "")
def dist = d['U']
def s1 = null
if(road){
s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","")
newAddress = newAddress + street + dist + road + s1
}else if(d['L']){
s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","")
newAddress = newAddress + street + dist + d['L'] + s1
}
d['V'] = newAddress
},0)
}
def run(){
@@ -18,6 +51,8 @@ class WuXiAddressTest {
//
// }
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
if(!d['S']){
def locationStr = getLocationInfo(d['D'], d['Q'])
@@ -6,6 +6,7 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
@@ -47,14 +48,13 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-05-01"
static def statDate = "2021-08-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
}
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年9月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -308,10 +308,7 @@ class YouDaDataParser {
}
);
}
latch.await()
}
def saveEsfHouseToDB(def data){
@@ -367,15 +364,16 @@ class YouDaDataParser {
row.ydId = data[2].toInteger()
row.fjCity = city
def map = matchDistrict(city, row.region, row.districtName)
if(!map){
map = matchDistrict(city, row.region, row.address)
}
// def map = matchDistrict(city, row.region, row.districtName)
// if(!map){
// map = matchDistrict(city, row.region, row.address)
// }
def map = SeaweedDataMatchProcess.doMatch(city, row.region, row.districtName, row.address);
if(map){
row.fjRegion = map.region
row.fjName = map.name
row.lat = map.lat
row.lng = map.lng
row.lat = map.lat.toDouble()
row.lng = map.lng.toDouble()
row.location = map.lat.toString()+","+map.lng.toString()
def obj = CommonDatautils.findDistrictById(map.id.toString());
@@ -7,6 +7,13 @@ import java.util.regex.Pattern
class CcbCodeUtils {
//否:0 是:1
static def parseFlag(String flag){
if(flag && flag == "是"){
return 1;
}
return 0;
}
static def parseBuildYear(String buildDate) {
if(buildDate){
@@ -53,6 +60,44 @@ class CcbCodeUtils {
return null
}
// 高层塔楼 0101 增加“超高层” 0108
// 高层板楼 0102 增加“老洋房” 0109
// 小高层板楼 0103 增加“花园洋房” 0110
// 多层板楼 0104 增加“四合院” 0111
// 独栋别墅 0105 增加“老式别墅洋房” 0112
// 联排别墅 0106
// 叠拼别墅 0107
// 其他建筑形式 0199
static def parseBuildType(def buildType){
if(buildType){
if(buildType.contains("老式别墅洋房")){
return "0112"
}else if(buildType.contains("四合院")){
return "0111"
}else if(buildType.contains("花园洋房")){
return "0110"
}else if(buildType.contains("老洋房")){
return "0109"
}else if(buildType.contains("叠拼")){
return "0107"
}else if(buildType.contains("联排")){
return "0106"
}else if(buildType.contains("独栋")){
return "0105"
}else if(buildType.contains("多层")){
return "0104"
}else if(buildType.contains("超高层")){
return "0108"
}else if(buildType.contains("小高层")){
return "0103"
}else if(buildType.contains("高层")){
return "0102"
}
}
return "0199"
}
//朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399
static def getDirectionCode(def direction){
@@ -161,4 +206,19 @@ class CcbCodeUtils {
}
return 0;
}
//房屋用途
static def parseHouseUse(String houseUse){
if(houseUse){
if(houseUse.contains("住宅")){
return 11;
}else if(houseUse.contains("商住")){
return 13;
}else{
return 99
}
}
return 99
}
}
@@ -0,0 +1,18 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
class CcbXykData {
static main(args) {
def task = new CcbXykData()
task.run()
}
def run(){
def districts = CommonDatautils.findAllDistrictByCity("ÉϺ£","district",["city","region","name","estate_charge"])
for(def d : districts){
}
}
}
@@ -3,35 +3,45 @@ package com.fangjia.script.gj.ccb
import org.apache.commons.lang.StringUtils
class FileUtils {
static def batchCreateFile(ret, fileName, headers,charset){
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
File file = new File(fileName);
if (file.exists()) {
file.delete();
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName);
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
if(headers){
sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(System.getProperty("line.separator"));
println index
if(row){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(System.getProperty("line.separator"));
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
@@ -0,0 +1,465 @@
package com.fangjia.script.gj.data
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class AddDistrictFieldValue {
def detailColl = CrawalDBUtil.getDistrictDetailColl();
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
def task = new AddDistrictFieldValue();
task.run();
}
def run(){
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){
// def rets = findAllDataByCity(city)
// match(rets)
// }
//第三方均价数据填补
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
// def rMap = getMatchDistrict(city)
// createResultValue(dists, rMap)
// }
//
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
// addFinishDate(dists)
addFieldValueByHouse(dists)
}
}
def match(def rets){
def totalCount = new AtomicLong()
def total = rets.size()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<rets.size();i++){
final def d = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def region = RegionData.formatRegion(d.city, d.region)
if(region) {
def rMap = SeaweedDataMatchProcess.doMatch(d.city, region, d.name, d.address);
if(rMap){
def q = new BasicDBObject()
q.put("_id", d._id)
def udata = new BasicDBObject()
udata.put("match_city", rMap.city)
udata.put("match_region", rMap.region)
udata.put("match_name", rMap.name)
detailColl.update(q, new BasicDBObject('$set',udata))
}
println totalCount.get()+"/${total}"+" "+d.city +" "+d.region+" "+d.name +" MATCH:"+rMap
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def findAllDataByCity(def city){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
keys.put("address", 1)
keys.put("match_name", 1)
def dbcursor = detailColl.find(new BasicDBObject("city", city), keys).batchSize(2000)
def rets = []
for(def d : dbcursor){
if(!d.match_name){
rets << d
}
}
return rets;
}
def getMatchDistrict(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("match_name", new BasicDBObject('$exists', true))
def dbcursor = detailColl.find(q)
def rMap = [:]
for(def d : dbcursor){
def key = d.match_city +"#"+d.match_region+"#"+d.match_name
def sourceMap = rMap.get(key)
if(!sourceMap){
sourceMap = [:]
rMap.put(key, sourceMap)
}
def value = sourceMap.get(d.root)
if(!value){
sourceMap.put(d.root, d)
}
}
return rMap;
}
def addFinishDate(dists){
def updateIndex = 0
def total = dists.size()
for(def d : dists){
if(!d.finish_building_date){
def udata = new BasicDBObject()
def newValue = FiledStrand.processFinishBuildingDate(d.live_date)
if(!newValue && d.open_date){
def date = d.open_date
newValue = date.getYear()+1900
}
if(!newValue && d.build_date){
newValue = FiledStrand.processFinishBuildingDate(d.build_date)
}
if(newValue){
udata.put("finish_building_date", newValue)
}
if(udata.size()>0){
updateIndex++
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
def createResultValue(dists, rMap){
def updateIndex = 0
def total = dists.size()
for(def d : dists){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
if(sourceMap){
def udata = new BasicDBObject()
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
udata.put("all_area", lastValue)
}
}
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
udata.put("build_area", lastValue)
}
}
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
udata.put("virescence_percent", lastValue)
}
}
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
udata.put("cubage_percent", lastValue)
}
}
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
udata.put("finish_building_date", lastValue)
}
}
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
udata.put("household_count", lastValue)
}
}
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
udata.put("car_site_count", lastValue)
}
}
if(udata.size()>0){
updateIndex++
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
def getValuesByField(sourceMap, field, estateType){
def map = [:]
sourceMap.each{k,v->
def oldValue = v.get(field);
if(oldValue){
def newValue = null;
if(["address"].contains(field)){
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
}
if(["all_area","build_area"].contains(field)){
newValue = FiledStrand.processArea(oldValue)
}
if(["build_date"].contains(field)){
newValue = FiledStrand.processFinishBuildingDate(oldValue)
}
if(["house_count"].contains(field)){
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
}
if(["greening_rate"].contains(field)){
newValue = FiledStrand.processVirescencePercent(oldValue)
}
if(["volumetric_rate"].contains(field)){
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
}
if(["car_site_desc","car_site"].contains(field)){
newValue = FiledStrand.processCarSite(oldValue)
}
if(newValue){
def count = map.get(newValue)
if(!count){
map.put(newValue, 1)
}else{
map.put(newValue, count+1)
}
v.put(field, newValue)
}
}
}
def lastValue = null
if(map && map.size()>0){
map = map.sort {a, b -> b.value <=> a.value}
def values = map.values()
if(values.size()==1){
lastValue = map.keySet()[0]
}else{
def count1 = map.values()[0]
def count2 = map.values()[1]
if(count1 > count2){
lastValue = map.keySet()[0]
}else{
def rets = [map.keySet()[0],map.keySet()[1]]
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
for(def root : roots){
def data = sourceMap.get(root)
if(data){
def value = data.get(field)
if(rets.contains(value)){
lastValue = value
break
}
}
}
}
}
}
return lastValue;
}
def addFieldValueByHouse(def dists){
def updateIndex = new AtomicLong()
def totalCount = new AtomicLong()
def total = dists.size()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(d.estate_type2!="别墅" && (!d.finish_building_date || !d.build_type)){
def udata = new BasicDBObject()
def retMap = findHouses(d.city, d.region, "sell", d.name, d.alias)
if(retMap){
println retMap
if(!d.finish_building_date && retMap.containsKey("build_date")){
def dateMap = retMap.get("build_date")
if(dateMap){
def value = dateMap.keySet()[0]
if(value){
udata.put("finish_building_date", value)
}
}
}
if(!d.build_type && retMap.containsKey("height")){
def heightMap = retMap.get("height")
if(heightMap){
def heights = heightMap.keySet()
if(heights){
def max = heights.max()
if(max<=6){
udata.put("build_type", "多层")
}else if(max>6 && max<=11){
udata.put("build_type", "小高层")
}else if(max>11 && max<30){
udata.put("build_type", "高层")
}else if(max>30){
udata.put("build_type", "超高层")
}
}
}
}
}
if(udata.size()>0){
updateIndex.addAndGet(1)
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex.get() +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}else{
println totalCount.get() +" "+d.city +" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
latch.await(5, TimeUnit.MINUTES);
}
def findHouses(def city, def region, def cat, def name, def alias){
// QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
// must(QueryBuilders.termQuery("cat", cat)).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def restMap = [:]
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(seawater.build_date){
def newValue = FiledStrand.processFinishBuildingDate(seawater.build_date)
if(newValue){
def valueMap = restMap.get("build_date")
if(!valueMap){
valueMap = [:]
restMap.put("build_date", valueMap)
}
def count = valueMap.get(newValue)
if(!count){
valueMap.put(newValue, 1)
}else{
valueMap.put(newValue, count + 1)
}
}
}
if(seawater.height){
def hightMap = restMap.get("height")
if(!hightMap){
hightMap = [:]
restMap.put("height", hightMap)
}
def count = hightMap.get(seawater.height)
if(!count){
count = 1
hightMap.put(seawater.height, count)
}else{
hightMap.put(seawater.height, count + 1)
}
}
}
}
if(restMap){
def dateMap = restMap.get("build_date")
if(dateMap){
dateMap = dateMap.sort {a, b -> b.value <=> a.value}
restMap.put("build_date", dateMap)
}
}
return restMap;
}
}
@@ -0,0 +1,186 @@
package com.fangjia.script.gj.data
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
class BestDistrictPriceExport {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new BestDistrictPriceExport();
//task.run()
task.run2()
}
def run2(){
def filePath = "d:\\数据\\广发\\楼盘.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
def stat = findDistrictPrice(row['A'], row['B'], row['D'], "20210801")
if(stat){
row['S'] = stat.avg_price
}
println row['A']+" "+row['B']+" "+ row['C']+" "+row['S']
})
}
def run(){
def city = "佛山"
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
def rets = []
rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"]
def distMap = findCreditHightDistrict2(city, "20210711");
distMap.each{region, dists->
if(region=="三水"){
def addIndex = 0
for(def dist : dists){
if(addIndex <=30){
def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name)
def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711")
if(otherPrice){
def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def ret = []
ret << "佛山"
ret << dist.region
ret << dist.district_name
ret << dist.estate_type
for(def weekend : months){
def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend)
if(obj){
ret << obj.avg_price
}else{
ret << ""
}
}
ret << otherPrice.lianjia_esf_price
ret << stat.credit
rets << ret
addIndex++
}
}
}
}
}
}
ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格")
}
def findDistrictPrice(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
def obj = DIST_SELL_STAT.findOne(q)
return obj
}
def findCreditHightDistrict(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def typeMap = retMaps.get(d.region)
if(!typeMap){
typeMap = [:]
retMaps.put(d.region, typeMap)
}
def rets = typeMap.get(d.estate_type)
if(!rets){
rets = []
typeMap.put(d.estate_type, rets)
}
def count = 0
typeMap.each {k,v->
if(v){
count = count + v.size()
}
}
if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){
def m = [:]
m.city = d.city
m.region = d.region
m.district_name = d.district_name
rets << m
}else if(rets.size() <= 20){
def m = [:]
m.city = d.city
m.region = d.region
m.district_name = d.district_name
rets << m
}
def total = 0
retMaps.each {k,m->
m.each {t,v->
total = total + v.size()
}
}
if(total>100){
break;
}
}
def rows = []
retMaps.each {k,m->
m.each {t,v->
rows.addAll(v)
}
}
return rows
}
def findCreditHightDistrict2(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
def m = [:]
m.city = d.city
m.region = d.region
m.estate_type = d.estate_type
m.district_name = d.district_name
rets << m
}
//
// def rows = []
// retMaps.each {k,v->
// rows.addAll(v)
// }
return retMaps;
}
def findNearThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", 202106)
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
return AVERAGE_STATISTICS.findOne(q)
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"]
def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"]
def rets = []
rets << headers
for(def s : districts){
def arrs = s.split("_")
def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2])
def ret = []
ret << s
if(m){
ret << m.city
ret << m.district
ret << m.name
if(m.alias){
ret << m.alias.toString().replace("[","").replace("]","")
}else{
ret << ""
}
ret << m.address
if(m.addressAlias){
ret << m.addressAlias.toString().replace("[","").replace("]","")
}else{
ret << ""
}
ret << m.avgPrice
ret << m.lng
ret << m.lat
ret << m.block
ret << m.loopLine
ret << m.propertyType
ret << m.buildingType
ret << m.commercialArea
ret << m.totalUnitCount
ret << m.managementFee
ret << m.parking
ret << m.parkingDescription
ret << m.ownershipPeriod
ret << m.ownershipType
ret << m.startBuildTime
ret << m.startSellingTime
ret << m.finishBuildTime
ret << m.moveinTime
ret << m.floorSpace
ret << m.constructionSpace
ret << m.plotRatio
ret << m.greenPercentage
ret << m.preSalePermit
ret << m.developer
ret << m.propertyManagement
ret << m.decorationStandard
ret << m.buildingStructure
ret << m.supportingFacilities
ret << m.transportation
ret << m.relatedInfo
ret << m.description
ret << m.salesOfficeAdd
ret << m.salesPhoneNumber
ret << m.discount
ret << m.buildingCount
ret << m.finishYear
}
rets << ret
}
ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据")
@@ -14,12 +14,45 @@ class FieldStat {
static main(args) {
def task = new FieldStat();
// task.run()
task.run1()
// task.run1()
// task.run2()
task.run3()
}
def run3(){
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
def rets = []
rets << ["城市","有地址别名总量","小区总量","有地址别名小区占比"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "addr_alias"]);
def alias_index = 0
def alias_total = 0
for(def d : districts){
def alias = d.addr_alias
if(alias){
alias_index++
}
}
def precent = 0
if(districts.size()){
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
}
rets << [city, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\全国城市地址别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
}
def run1(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -10,22 +11,23 @@ import com.mongodb.DBCollection
class RegionPriceExport {
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPriceExport();
task.run()
}
def run(){
def months = [202010,202011,202012,202101,202102,202103]
def regions = CommonUtils.getAllRegions("福州","region")
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
def regions = CommonUtils.getAllRegions("佛山","region")
def rets = []
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
regions.each{region,values->
def data = []
data << "福州"
data << "佛山"
data << region
for(def month :months){
def dbo = findPrice("福州", region, month);
def dbo = findWeekendPrice("佛山", region, month);
if(dbo){
data << dbo.avg_price
}else{
@@ -34,7 +36,7 @@ class RegionPriceExport {
}
rets << data
}
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格")
}
def findPrice(city, region, month){
@@ -45,4 +47,14 @@ class RegionPriceExport {
def dbo = REGION_SELL_STAT.findOne(q)
return dbo
}
def findWeekendPrice(city, region, month){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("cat", "区域")
q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
def dbo = REGION_STAT.findOne(q)
return dbo
}
}
@@ -24,10 +24,10 @@ class DataProcess {
static main(args) {
//建行信用卡城市(临汾--晋城).xlsx
def filePath = "D:\\数据\\建行银行\\建行信用卡入库小区V1.xlsx"
def filePath = "D:\\数据\\小区\\地图\\兰州.xlsx"
def task = new DataProcess();
// task.batchSaveDataToDB(filePath)
// task.mapBatchSaveDataToDB(filePath)
task.mapBatchSaveDataToDB(filePath)
// task.calcutePrice(filePath);
// task.calcutePrice2();
// task.deleteRepDist("株洲");
@@ -35,7 +35,7 @@ class DataProcess {
// task.standartName(filePath);
task.saveGaoDeMap(filePath);
// task.saveGaoDeMap(filePath);
// task.updateDBAddress();
// task.updateRegion()
@@ -120,13 +120,10 @@ class DataProcess {
def lastDatas = []
if(datas){
def keys = []
for(def data : datas){
if(data['A']=="厦门" && data['I'] && data['I'].toString().startsWith("24")){
for(def data : datas){//20,21,25,
if(data['I'] && [20,21,26,27].contains(data['I'].toString().toDouble().toInteger())){
def key = data['A']+"##"+data['B']+"##"+data['H']
if(!keys.contains(key) && data['H'].size()>2){
// if(data['I'].toInteger()==22){//&& data['N']<3
// continue;
// }
if(!keys.contains(key) && data['H'] && data['H'].size()>2){
keys << key
lastDatas << data
}
@@ -168,7 +165,7 @@ class DataProcess {
addIndex++
def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null)
println "${addIndex}【新增入库】"+ values
// FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
}else{
println "【已存在】 ${d['A']} ${d['B']} ${d['H']}"
}
@@ -1,8 +1,11 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
@@ -13,7 +16,7 @@ class ExportData {
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -26,6 +29,7 @@ class ExportData {
def task = new ExportData();
task.exportThird()
// task.exportMap();
}
def exportMap(){
@@ -51,19 +55,18 @@ class ExportData {
//"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特"
//宁夏 ----- 中卫 固原 石嘴山 吴忠 银川
//青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁
def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
"中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
"防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
//"沈阳","大连","抚顺","鞍山"
//"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆"
//"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛",
//"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州",
//"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城",
// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"];
for(def city : citys){
def path = "D:\\数据\\小区\\地图\\${city}.xlsx";
def datas = findAllDatas(city, null)
println "${city}共计:"+datas.size()
ExcelUtils.write(path, datas, "数据")
}
def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx";
def datas = findAllDatas(citys, null)
println "共计:"+datas.size()
ExcelUtils.write(path, datas, "数据")
}
@@ -71,7 +74,7 @@ class ExportData {
def exportThird(){
def citys = ["重庆"]
def citys = ["南昌"]
for(def city : citys){
def rets = []
rets << ["城市","区域","名称","别名"]
@@ -98,9 +101,9 @@ class ExportData {
def findAllDatas(def citys, def pname){
def findAllDatas(def city, def pname){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1")
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'")
// sqlBuffer.append(" and pname = '"+pname+"'")
// sqlBuffer.append(" and cityname = '"+pname+"'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
@@ -113,16 +116,19 @@ class ExportData {
if(source && source=="高德"){
println index
def type = rs.getObject("type")
def city = rs.getString("cityname")
if(citys.contains(city) && type.contains("商务住宅")){
//def city = rs.getString("cityname")
if(type.contains("住宅")){
def ret = []
ret << city
ret << rs.getString("adname")
ret << rs.getString("name")
ret << rs.getString("address")
ret << rs.getString("type")
ret << rs.getFloat("location.lon")
ret << rs.getFloat("location.lat")
def loaction = rs.getObject("location")
if(loaction){
ret << DataHandleUtils.getDoubleRound(6, loaction.x)
ret << DataHandleUtils.getDoubleRound(6, loaction.y)
}
rets << ret
}
}
@@ -68,13 +68,16 @@ class FangjiaClientUtils {
public static def addressStand(def address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
def content = new HttpRequest().newGet(url, null, "utf-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
@@ -1,51 +1,65 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import com.mongodb.BasicDBObject
import java.util.regex.Pattern
/***
* 地图小区处理
*/
class MapDistrictProcess {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) {
def task = new MapDistrictProcess();
task.run()
// task.updateRegion()
}
/**
* 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路
* 20ㄩ祥肮Е郖陔崝ㄗ踱笢拸ㄘ 21ㄩ煦ヽ陔崝 22:煦杅婓0.6眕奻
* 20:不同区域新增(库中无) 21:分期新增 23:不同区域,距离相近 25:无匹配,可新增 22:有匹配,距离1000米以下,相似度在0.6以上 24:疑似匹配有误 27:人工判定入库
* @return
*/
def run(){
//"蔬劼華芞.xlsx","涳蔬華芞.xlsx","腦膘華芞.xlsx","蔬昹華芞.xlsx","刓陲華芞.xlsx","假閣華芞.xlsx",
//"奻漆華芞.xlsx","控儔華芞.xlsx"
//"假閣華芞.xlsx","蔬劼華芞.xlsx","涳蔬華芞.xlsx","腦膘華芞.xlsx","蔬昹華芞.xlsx","刓陲華芞.xlsx",
//"笭ヰ華芞.xlsx","毞踩華芞.xlsx",
//"奻漆華芞.xlsx","控儔華芞.xlsx",
//"碩控華芞.xlsx","碩鰍華芞.xlsx","綬控華芞.xlsx","綬鰍華芞.xlsx","嫘昹華芞.xlsx",
//"憚輿華芞.xlsx","賽譴華芞.xlsx""賽譴華芞.xlsx","憚輿華芞.xlsx","窪韓蔬華芞.xlsx","刓昹華芞.xlsx","漆鰍華芞.xlsx"]
def files = ["ヤ漆華芞.xlsx","譴狦華芞.xlsx","囀蟹嘉華芞.xlsx","堁鰍華芞.xlsx","匟昹華芞.xlsx","陔蔭華芞.xlsx","侐捶華芞.xlsx"]
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
//"乌鲁木齐.xlsx","烟台.xlsx","保定.xlsx","贵阳.xlsx","湖州.xlsx","惠州.xlsx","吉林.xlsx","遵义.xlsx","宜昌.xlsx","芜湖.xlsx","兰州.xlsx"
def files = ["兰州.xlsx"]
for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
typeFilter(filePath)
nameStandard(filePath);
dataFilter(filePath)
matchDistDatas(filePath)
houseCalculate(filePath);
////// //苀數岆瘁湔婓嗣跺Е郖
//统计是否存在多个区域
matchDistDatas(filePath)
moreRegionCalculate(filePath)
//
addFlag(filePath)
mapMatch(filePath);
baiduTypeFilter(filePath);
baiduMap(filePath);
// baiduMap(filePath);
distanceFilter(filePath)
predictType(filePath)
noMatchFlag(filePath)
matchFlag(filePath)
addFeature(filePath)
houseMatch(filePath);
predictType(filePath)
// nameScore(filePath);
nameScore(filePath);
thirdPartyData(filePath)
}
}
@@ -59,7 +73,12 @@ class MapDistrictProcess {
row['I'] = 1
}else{
if(name.toString().length()<3){
row['I'] = 18
if(row['N'] && row['N']>=10 && !row['J']){
row['I'] = 25
}else{
row['I'] = 18
}
}else{
def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){
@@ -74,7 +93,7 @@ class MapDistrictProcess {
}
});
println "2.靡備徹薦俇傖"
println "3.名称过滤完成"
}
def typeFilter(filePath){
@@ -88,15 +107,30 @@ class MapDistrictProcess {
println "1.类型过滤完成"
}
def baiduTypeFilter(filePath){
def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"]
ExcelUtils.multiThreadingModify(filePath, { row ->
def type = row['S']
if(type && !types.contains(type)){
if(type && row['Q'] && row['Q']==row['H'] && !types.contains(type)){
row['I'] = 1
}
});
println "1.濬倰徹薦俇傖"
println "9.百度类型过滤完成"
}
def distanceFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']){
if(row['X'] && row['Y']){
if(row['X']<=500 && row['Y']>=0.5){
row['I'] = 12
}
}
}
});
println "10.距离过滤完成"
}
def nameStandard(filePath){
@@ -109,7 +143,7 @@ class MapDistrictProcess {
}
}, 0);
println "1.靡備梓袧趙俇傖"
println "2.名称标准化完成"
}
def matchDistDatas(def path){
@@ -117,17 +151,29 @@ class MapDistrictProcess {
ExcelUtils.multiThreadingModify(path,{ List row ->
index ++
if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],);
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
row['J'] = rMap.city
row['K'] = rMap.region
row['L'] = rMap.name
row['M'] = rMap.address
if(rMap.lat){
def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
def maplng = baidu.x
def maplat = baidu.y
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
if(score<0.3){
score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['M'])
}
row['Y'] = score
}
}
println index +" "+row
}
},0)
println "3.苤Е⑤饜俇傖"
println "5.小区匹配完成"
}
def houseCalculate(filePath){
@@ -136,6 +182,7 @@ class MapDistrictProcess {
row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H'])
}
});
println "4.房源统计完成"
}
def moreRegionCalculate(filePath){
@@ -156,8 +203,12 @@ class MapDistrictProcess {
}
}
if(!regions.contains(row['B'])){
row['O'] = regions.toString()
row['I'] = 20
if(row['X'] && row['X']<=500 && row['Y'] && row['Y']>=0.5){
row['I'] = 23
}else{
row['O'] = regions.toString()
row['I'] = 20
}
}else if(regions.contains(row['B'])){
row['O'] = "已存在"
row['I'] = 10
@@ -165,11 +216,12 @@ class MapDistrictProcess {
}
}
});
println "6.区域过滤完成"
}
def addFlag(def filePath){
def index = 0
ExcelUtils.modify(filePath, { row ->
ExcelUtils.multiThreadingModify(filePath, { row ->
index++
if(!row['I']) {
//根据别名查找
@@ -201,6 +253,8 @@ class MapDistrictProcess {
println index + " " + row
}
});
println "7.打标记完成"
}
def baiduMap(def filePath){
@@ -228,6 +282,7 @@ class MapDistrictProcess {
}
}
});
println "11.特征过滤完成"
}
def mapMatch(def filePath){
@@ -255,16 +310,22 @@ class MapDistrictProcess {
println index +" "+row
},0);
println "4.華芞⑤饜俇傖"
println "8.地图匹配完成"
}
def houseMatch(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) {
def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
if(rMap){
row['I'] = rMap.flag
}
// def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
// if(rMap){
// row['I'] = rMap.flag
// }
if(row['J']){
if(row['X'] && row['Y'] ){
}
}
}
},0);
}
@@ -309,6 +370,41 @@ class MapDistrictProcess {
},0);
}
def noMatchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && !row['J']) {
if(row['N']>0){
row['I'] = 25
}else{
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 25
}
}
}
});
}
def matchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && row['J'] && row['X'] && row['Y']) {
if(row['X']<=1000 && row['Y']>=0.6){
row['I'] = 22
}else if(row['X']<=600 && row['Y']>=0.5){
row['I'] = 22
}else if(row['X']<=1300 && row['Y']>=0.8){
row['I'] = 22
}else{
row['I'] = 24
}
}else if(!row['I'] && row['J'] && row['Y']==0){
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 26
}
}
});
}
def nameScore(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) {
@@ -338,4 +434,59 @@ class MapDistrictProcess {
}
},0);
}
def updateRegion(){
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx";
ExcelUtils.multiThreadingModify(filePath, { row ->
def region = getRegionByLocation(row['F'], row['G']);
if(region){
row['B'] = RegionData.formatRegion(row['A'], region)
}
},0)
}
def thirdPartyData(filePath){
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['I'] && [20,21,25,26,27].contains(row['I'].toString().toDouble().toInteger())){
def rets = findIsExistsOtherWebsite(row['A'], row['B'], row['H'])
if(rets){
row['W'] = rets.join(",")
}
}
},0)
}
def findIsExistsOtherWebsite(def city, def region, def name){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name", name)
def dbcursor = districtDetailColl.find(q)
def rets = []
for(def r : dbcursor){
if(!rets.contains(r.root)){
rets << r.root
}
}
return rets
}
def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
if(jsonObject && jsonObject.info=="OK"){
def compent = jsonObject.regeocode.addressComponent
if(compent){
def township = compent.township
return township;
}
}
return null
}
}
@@ -7,7 +7,9 @@ import java.util.regex.Pattern
class NameProcess {
static def filterName(def name){
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心","养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会"]
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心",
"养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房",
"合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼","养生"]
def isAdd = true;
for(def key : keys){
if(name.contains(key)){
@@ -17,7 +19,7 @@ class NameProcess {
}
if(isAdd){
def endKeys = ["路","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍"]
def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门","学校","公司"]
for(def key : endKeys){
if(name.toString().endsWith(key)){
isAdd = false;
@@ -27,7 +29,7 @@ class NameProcess {
}
if(isAdd){
def startKeys = ["FQ"]
def startKeys = ["FQ","社区"]
for(def key : startKeys){
if(name.toString().endsWith(key)||name.toString().startsWith(key)){
isAdd = false;
@@ -36,7 +38,7 @@ class NameProcess {
}
}
if(isAdd){
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房"]
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房", "家属宿舍楼","卫生生活区"]
for(def str : strs){
if(str == name){
isAdd = false;
@@ -56,7 +58,7 @@ class NameProcess {
}
}
//.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号")
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","")
if(name && name.contains("区")){
def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区"))
@@ -105,7 +107,7 @@ class NameProcess {
}
}
if(name && !name.contains("路") && !name.contains("街") && !name.contains("大道") && !name.contains("巷")){
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(号|排)"))
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(排)"))
if(h1){
name = name.replaceAll(h1,"")
}
@@ -191,12 +193,12 @@ class NameProcess {
if(str == name){
return null
}
if(name){
s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
if(s1){
return null
}
}
// if(name){
// s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
// if(s1){
// return null
// }
// }
return name.toUpperCase()
}
@@ -15,36 +15,44 @@ class SeaweedDataMatchProcess {
if(!rMap.city){
def obj = CommonDatautils.findDistrict(city, region, name, "district")
if(!obj){
obj = CommonDatautils.findDistrictByAlias(city, region, name, "district")
def datas = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if(datas){
obj = datas[0]
}
}
if(obj){
rMap.estate_type2 = obj.estate_type2
rMap.city = obj.city
rMap.region = obj.region
rMap.alias = obj.alias
rMap.name = obj.name
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
rMap.id = obj._id.toString()
}
}
}
if(!rMap && address){
// if (findPattern(address)){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
// }
}
if(!rMap && name){
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
}
if(rMap && rMap.id && region.contains(rMap.region)) {
def obj = CommonDatautils.findDistrictById(rMap.id.toString());
if (obj) {
rMap.estate_type2 = obj.estate_type2
if(!rMap && address){
def flag = address.matches('^[\u4e00-\u9fa5]+?(·|´óµÀ|½Ö).+[ŪºÅ]$')
if(flag){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
}
}else{
rMap = null
}
// if(rMap && rMap.id && region.contains(rMap.region)) {
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
// if (obj) {
// rMap.estate_type2 = obj.estate_type2
// }
// }else{
// rMap = null
// }
if(!rMap && name && name.contains("¡¤")){
def newName = trimChar(name).replace("¡¤","")
rMap = matchFromDB2(city,newName)
@@ -0,0 +1,265 @@
package com.fangjia.script.gj.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.lucene.queryparser.classic.QueryParser
class SourceDistDataMatch {
static main(args) {
def task = new SourceDistDataMatch();
// task.run()
task.mapRun()
// task.filterName()
// task.filterType()
}
def run(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(obj){
d['R'] = 1
}else{
def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709)
if(distCount){
d['W'] = distCount
}
}
},0)
}
def mapRun(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['N'] && !d['F']){
def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']]
def locationMap = matchMap(dist)
if(locationMap){
d['F'] = locationMap.source
d['G'] = locationMap.region
d['H'] = locationMap.mapName
d['I'] = locationMap.address
d['J'] = locationMap.type
d['K'] = locationMap.lng
d['L'] = locationMap.lat
d['M'] = locationMap.score
}
}
},0)
}
def filterName(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['N']) {
def name = d['C']
if (name) {
def flag = NameProcess.filterName(name)
if (!flag) {
d['N'] = 4
}
}
}
},0)
}
def filterType(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['N']){
def type = d['J']
if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){
}else if(type && type.contains("商务住宅") && !type.contains("商住两用")){
def count = 0
if(d['S'] && d['S']!='null'){
count = d['S'].toDouble().toInteger()
}
if(count<10){
d['N'] = 3
}
}else{
if(type){
d['N'] = 3
}
}
}
},0)
}
def matchMap(def dist){
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
if(!gaodeRest && dist.address){
gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德")
}
def gdlocationMap = null
def bdlocationMap = null
if(gaodeRest){
gdlocationMap = getMapLocation2(dist, gaodeRest);
if(gdlocationMap){
gdlocationMap.source = "高德"
}
}
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
if(baiduRest){
bdlocationMap = getMapLocation2(dist, baiduRest);
if(bdlocationMap){
bdlocationMap.source = "百度"
}
}
if(gdlocationMap && bdlocationMap){
if(gdlocationMap.score>=bdlocationMap.score){
return gdlocationMap
}else{
return bdlocationMap
}
}else if(gdlocationMap){
return gdlocationMap
}else if(bdlocationMap){
return bdlocationMap
}
return null
}
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
def aliasStr = null;
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
def getMapLocation2(def dist, def rets){
def data1 = findSimilarData(dist, rets, true)
def data2 = findSimilarData(dist, rets, false)
println data1
println data2
if(data1 && data1.score>1){
return data1
}else if(data2 && data2.score>1){
return data2
}
return null
}
def findSimilarData(def dist, def rets, def regionImpr){
def lat = null
def lng = null
def region = null
def mapName = null
def address = null
def score = 0
def type = null
def maxScore = 0
def standardName = null
for(def r : rets){
if(!r.city){
continue
}
if(!r.city.contains(dist.city)){
continue
}
if(["公交车站"].contains(r.type)){
continue
}
if(regionImpr && !r.district){
continue
}
if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){
continue
}
def name = nameReplace(r.name, dist.city)
def distName = nameReplace(dist.name, dist.city)
def score3 = 0.0
if(r.district && dist.region && r.district.contains(dist.region)){
score3 = 0.2
}
def score1 = 0.0;
def score2 = 0.0
if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){
if(r.address){
score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2
}
}else{
score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2;
if(dist.address && r.address){
score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2
}
}
score3 = score1 + score2
if(r.type && r.type.contains("住宅")){
score3 = score3 + 0.2
}
if(score3 >maxScore){
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
standardName = name
address = r.address
type = r.type
score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10)
maxScore = score3
}
println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3
}
if(lng){
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName]
}
return null
}
def nameReplace(def s, city){
if(s.contains("(") && s.contains(")")){
def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")"))
if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){
s = s.toString().substring(0, s.indexOf("("))
}
}
def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim()
if(name.startsWith(city)){
def newName = name.replace(city+"市","").replace(city,"")
if(newName && newName=="花园"){
return name
}else{
name = newName
}
}
return name
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -49,7 +49,7 @@ class ZsPersonProcess {
// def loactionMap = task.matchMap(dist)
// println loactionMap
// task.matchMap()
task.matchMap()
// task.matchDb()
// task.matchDb2()
@@ -0,0 +1,320 @@
package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
class AddDetailDistrict {
def detailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) {
def task = new AddDetailDistrict();
// task.run();
// task.delRepeatDist
task.addEstateType()
}
def addEstateType(){
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","estate_type2"])
for(def d : districts) {
if(!d.estate_type2){
def estateType2 = "普通住宅"
if(d.name.contains("别墅")){
estateType2 = "别墅"
}else if(d.name.contains("商住")){
estateType2 = "商住"
}
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("estate_type2", estateType2).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
println city
}
}
def delRepeatDist(){
def citys = CcbCommonUtils.citys
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
def index = 0
for(def d : districts){
index++
def key = d.city +"##"+d.region+"##"+d.name
if(!keys.contains(key)){
keys << key
}else{
println index +" "+d.region+" "+d.name
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("status", 1).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
}
}
def run() {
def citys = ["沈阳", "哈尔滨", "吉林", "遵义", "淄博", "西宁", "青岛", "大连", "宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
for (def city : citys) {
def rets = findAllDataByCity(city)
def rMap = getMatchDistrict(city)
toDB(rets, rMap)
}
}
def toDB(def rets, def rMap){
def saveDistKeys = []
def total = rets.size()
def index = 0
def addIndex = 0
for(def d : rets){
index++
def newRegion = RegionData.formatRegion(d.city, d.region)
def name = NameProcess.nameStandard(d.city, newRegion, d.name)
if(!name){
continue
}
def flag = NameProcess.filterName(name)
def key = null
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
if(flag && !saveDistKeys.contains(key)){
def gdData = FangjiaClientUtils.getMapData(d.city, newRegion, name, d.address,"", "高德")
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(gdData){
if(!newRegion){
newRegion = RegionData.formatRegion(d.city, gdData.district)
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
}
}
if(key && gdData && gdData.lng){
def matchMap = SeaweedDataMatchProcess.doMatch(d.city, newRegion, name, d.address);
if(!matchMap){
addIndex++
if(gdData.lng){
def gmap = GeocodeUtils.getGaodeToBaidu(gdData.lng.toDouble(), gdData.lat.toDouble())
lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
lng = (gdData.lng.toDouble()* 10000000000).longValue()
lat = (gdData.lat.toDouble()* 10000000000).longValue()
}
def newAddress = DataUtil.formateAddress(d.city, d.address)
def udata = createFieldValue(d, rMap)
udata.put("lng", lng)
udata.put("lat", lat)
println index+"/${addIndex}/${total} "+d.city+" "+newRegion+" "+name+" "+d.root+" "+udata
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", name)
udata.put("address", newAddress)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("status", 0)
udata.put("cat", "district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
DbUtils.seaweed.save(udata)
saveDistKeys << key
}
}
// println d.city+" "+ d.region+" "+d.name
}
}
}
def createFieldValue(d, rMap){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
def udata = new BasicDBObject()
if(sourceMap){
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
udata.put("all_area", lastValue)
}
}
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
udata.put("build_area", lastValue)
}
}
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
udata.put("virescence_percent", lastValue)
}
}
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
udata.put("cubage_percent", lastValue)
}
}
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
udata.put("finish_building_date", lastValue)
}
}
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
udata.put("household_count", lastValue)
}
}
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
udata.put("car_site_count", lastValue)
}
}
}
return udata
}
def getValuesByField(sourceMap, field, estateType){
def map = [:]
sourceMap.each{k,v->
def oldValue = v.get(field);
if(oldValue){
def newValue = null;
if(["address"].contains(field)){
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
}
if(["all_area","build_area"].contains(field)){
newValue = FiledStrand.processArea(oldValue)
}
if(["build_date"].contains(field)){
newValue = FiledStrand.processFinishBuildingDate(oldValue)
}
if(["house_count"].contains(field)){
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
}
if(["greening_rate"].contains(field)){
newValue = FiledStrand.processVirescencePercent(oldValue)
}
if(["volumetric_rate"].contains(field)){
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
}
if(["car_site_desc","car_site"].contains(field)){
newValue = FiledStrand.processCarSite(oldValue)
}
if(newValue){
def count = map.get(newValue)
if(!count){
map.put(newValue, 1)
}else{
map.put(newValue, count+1)
}
v.put(field, newValue)
}
}
}
def lastValue = null
if(map && map.size()>0){
map = map.sort {a, b -> b.value <=> a.value}
def values = map.values()
if(values.size()==1){
lastValue = map.keySet()[0]
}else{
def count1 = map.values()[0]
def count2 = map.values()[1]
if(count1 > count2){
lastValue = map.keySet()[0]
}else{
def rets = [map.keySet()[0],map.keySet()[1]]
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
for(def root : roots){
def data = sourceMap.get(root)
if(data){
def value = data.get(field)
if(rets.contains(value)){
lastValue = value
break
}
}
}
}
}
}
return lastValue;
}
def getMatchDistrict(def city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("match_name", new BasicDBObject('$exists', true))
def dbcursor = detailColl.find(q)
def rMap = [:]
for(def d : dbcursor){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
if(!sourceMap){
sourceMap = [:]
rMap.put(key, sourceMap)
}
def value = sourceMap.get(d.root)
if(!value){
sourceMap.put(d.root, d)
}
}
return rMap;
}
def findAllDataByCity(def city){
def q = new BasicDBObject("city", city);
q.put("match_name", new BasicDBObject('$exists', false))
q.put("avg_price", new BasicDBObject('$gt', 0))
def dbcursor = detailColl.find(q).batchSize(2000)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -1,8 +1,9 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.datastore.FieldValidatorUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.DbUtils
@@ -19,9 +19,9 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
// task.houseRun()
task.houseRun()
task.statHouseUpdateCount()
// task.statHouseUpdateCount()
}
def run(){
@@ -99,37 +99,35 @@ class EsMonthHouseStat {
}
def houseRun(){
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
def city = d['B']
if(city == "湖州"){
def count = findHouseCount(city, 20210301, 20210601)
d['E'] = count
ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d->
def city = d['A']
def count = findHouseCount(city, 20190709, 20210709)
d['E'] = count
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
alias = dist.alias
}else{
alias << dist.name
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['D'] = precent+"%"
def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['F'] = precent+"%"
},0)
}
@@ -0,0 +1,453 @@
package com.fangjia.script.gj.guangfa
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class GuangfaBlankSelectDistrict {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new GuangfaBlankSelectDistrict();
// task.run()
task.run2()
}
def run(){
def file = "D:\\数据\\广发\\获取楼盘信息需求(东莞和杭州).xlsx"
def cityMap = standardRegion(file)
def priceMap = getZhiDaoPrice(file)
selectDistricts(cityMap, priceMap)
println "写入完成"
}
def run2(){
selectDistricts2()
println "写入完成"
}
def selectDistricts(cityMap, priceMap){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
cityMap.each{city, regions->
if(["西安","东莞","杭州"].contains(city)){
def thirdPriceMap = findNearThirdPartyPrice(city, regions)
def retMaps = findCreditHightDistrict2(city, regions, "2021-08-01", priceMap, thirdPriceMap)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
//基本情况合格的小区
def distMap = [:]
for(def d : districts){
if(regions.contains(d.region)){// && d.household_count && d.household_count>=50
def distKeys = retMaps.get(d.region)
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
if(d.finish_building_date){
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
if(year){
def mod = 2021-year
if(mod <=15){
d.house_year = mod
}else{
d.house_year = null
}
}else{
d.house_year = null
}
}else{
d.house_year = null
}
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// if(d.finish_building_date){
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
// if(year){
// def mod = 2021-year
// if(mod <=15){
// def distKeys = retMaps.get(d.region)
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
// d.house_year = mod
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
// }
// }
// }
}
}
println city +" 基本情况合格的小区:"+distMap.size()
for(def r : regions){
def names = retMaps.get(r)
if(names){
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
rets.addAll(results)
}
}
println "结束"
}
}
println "开始写入Excel....."+rets.size()
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def selectDistricts2(){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
def citys = ["东莞","杭州"]
citys.each{city->
def thirdPriceMap = findNearThirdPartyPrice(city)
def retMaps = findCreditHightDistrict3(city, "2021-08-22", thirdPriceMap)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
//基本情况合格的小区
def distMap = [:]
for(def d : districts){
// && d.household_count && d.household_count>=50
def distKeys = retMaps.get(d.region)
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
if(d.finish_building_date){
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
if(year){
def mod = 2021-year
if(mod <=15){
d.house_year = mod
}else{
d.house_year = null
}
}else{
d.house_year = null
}
}else{
d.house_year = null
}
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// if(d.finish_building_date){
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
// if(year){
// def mod = 2021-year
// if(mod <=15){
// def distKeys = retMaps.get(d.region)
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
// d.house_year = mod
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
// }
// }
// }
}
println city +" 基本情况合格的小区:"+distMap.size()
for(def r : regions){
def names = retMaps.get(r)
if(names){
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
rets.addAll(results)
}
}
println "结束"
}
println "开始写入Excel....."+rets.size()
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(names.size())
for(def i=0;i<names.size();i++){
def key = names[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def d = distMap.get(key)
if(d){
def ret = getDetailInfo(d, priceMap)
if(ret){
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
// latch.await()
latch.await(5, TimeUnit.MINUTES);
return rets
}
def getDetailInfo(d, priceMap){
def retMap1 = null;
def retMap2 = null;
def retMap3 = null
if(d.city == "上海"){
retMap1 = findShTrades(d, "2021-01-01", "2021-07-01")
retMap2 = findShTrades(d, "2020-01-01", "2020-12-31")
retMap3 = findShTrades(d, "2019-01-01", "2019-12-31")
}else{
retMap1 = CcbCommonUtils.findTrades(d, "2021-01-01", "2021-07-01")
retMap2 = CcbCommonUtils.findTrades(d, "2020-01-01", "2020-12-31")
retMap3 = CcbCommonUtils.findTrades(d, "2019-01-01", "2019-12-31")
}
if(retMap1){
d.count1 = retMap1.count
d.avg_price1 = (int)retMap1.avg_price
}
if(retMap2){
d.count2 = retMap2.count
d.avg_price2 = (int)retMap2.avg_price
}
if(retMap3){
d.count3 = retMap3.count
d.avg_price3 = (int)retMap3.avg_price
}
if(!retMap1 || !retMap2 || !retMap3){
println d.city+" "+ d.region+" "+ d.name +"成交有一年为空过滤"
return null
}
def houses = EsSearchUtils.findSellHouseDetail(d.city, d.region, d.name)
def rooms = []
def areas = []
for(def h : houses){
rooms << h.room
areas << h.area
}
rooms = rooms.unique().sort()
areas = areas.unique().sort()
if(rooms){
d.room = rooms[0]+"-"+rooms[rooms.size()-1]
d.area = areas[0]+"-"+areas[areas.size()-1]
}
def zhidaoprice = priceMap.get(d.city+"#"+d.region)
def ret = [d.city, d.region, zhidaoprice, d.name, d.address, d.developer, d.cubage_percent, d.estate_company, d.household_count, d.house_year,d.room, d.area, d.count1, d.avg_price1,d.count2, d.avg_price2,d.count3, d.avg_price3]
println ret
return ret;
}
def standardRegion(filePath){
def city = null
def cityMap = new LinkedHashMap()
def index = 0
ExcelUtils.modify(filePath, { row ->
index ++
if(index >= 4){
if(row['C']){
city = row['C']
}
println city+" "+row['D']
def newRegion = null
if(city && row['D']){
newRegion = RegionData.formatRegion(city, row['D'])
row['F'] = newRegion
}
if(city && newRegion){
def regions = cityMap.get(city)
if(!regions){
regions = []
cityMap.put(city, regions)
}
if(!regions.contains(newRegion)){
regions << newRegion
}
}
}
});
return cityMap
}
def findCreditHightDistrict2(def city, def regions, def weekend, def priceMap, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
q.put("region", new BasicDBObject('$in', regions))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
if(regions.contains(d.region)){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice){
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def price = priceMap.get(d.city +"#"+d.region)
if(d.avg_price>price){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}else{
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}
return retMaps;
}
def getZhiDaoPrice(filePath){
def priceMap = [:]
def city = null
ExcelUtils.modify(filePath, { row ->
if(row['C']){
city = row['C']
}
if(city && row['F']){
def key = city +"#"+row['F']
if(!priceMap.containsKey(key)){
priceMap.put(key, row['E'])
}
}
})
return priceMap
}
public static def findShTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("fjCity", dist.city)
q.put("fjRegion", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("fjName", new BasicDBObject('$in', dist.alias))
}else{
q.put("fjName", dist.name)
}
q.put("contractTime", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = youda_esfhouse_coll.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.price){
total_price = total_price + d.price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
def findNearThirdPartyPrice(def city, regions){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", new BasicDBObject('$in', regions))
q.put("s_date", new BasicDBObject('$gte', 202105))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
def findNearThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', 202105))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
def findCreditHightDistrict3(def city, def regions, def weekend, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice){
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}else{
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
return retMaps;
}
}
@@ -0,0 +1,64 @@
package com.fangjia.script.gj.location
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class DistrictLocationUpdate {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new DistrictLocationUpdate()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
updateLocation(city, dists);
}
}
def queryPoiMap(def city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and source = '¸ßµÂ'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def retMap = [:]
while (rs.next()) {
def map = [:]
map.city = rs.getString("cityname")
map.region = rs.getString("adname")
map.name = rs.getString("name")
map.address = rs.getString("address")
map.type = rs.getString("type")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
def name = NameProcess.nameStandard(map.city, map.region, map.name)
if(name){
retMap.put(map.region+"##"+name, map)
}
}
return retMap;
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
@@ -14,7 +15,13 @@ import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern;
@@ -22,59 +29,81 @@ import java.util.regex.Pattern;
class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
Date date=new Date(1622802609);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String str = format.format(date);
System.out.println(str);
// Date date=new Date(1622802609);
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// String str = format.format(date);
// System.out.println(str);
// def task = new LocationTest()
def task = new LocationTest()
// task.run()
// task.run2()
// task.run3()
task.run3()
// task.run4()
// task.run5()
}
def run(){
def regionPointsMap = findRegionRange()
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
def city = "杭州"
def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0
def rets = []
for(def dist : districts){
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
if(!flag){
index++
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
def loactionMap = matchMap(dist)
if(loactionMap){
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if(dist.region==loactionMap.region){
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
if(!flag) {
index++
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
def loactionMap = matchMap(dist)
if (loactionMap) {
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if (dist.region == loactionMap.region) {
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
} else {
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
}
} else {
ret << ""
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,地图没找到"
}
if (ret) {
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}else{
ret << ""
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
}
rets << ret
}
});
}
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内")
latch.await()
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内")
}
def run2(){
@@ -133,10 +162,12 @@ class LocationTest {
def run3(){
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d->
def index = 0
ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d->
index++
def distance = null
if(d['G']){
distance = d['G'].toDouble()
if(d['H']){
distance = d['H'].toDouble()
}
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
@@ -180,12 +211,59 @@ class LocationTest {
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
def dbo = DbUtils.seaweed.findOne(q)
d['K'] = "修正"
}
}else if(d['E'] && d['B'] != d['E'] && distance && distance<=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
if(mapName.contains("-") && mapName.endsWith("楼")){
mapName = mapName.substring(0, mapName.indexOf("-"))
}
if(d['C']==mapName){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['E'])
q.put("name", d['C'])
q.put("cat", 'district')
q.put("status", 0)
def dbo = DbUtils.seaweed.findOne(q)
if(!dbo){
q.put("region", d['B'])
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(d['D']=="高德"){
lng = (d['I'].toDouble()*10000000000).longValue()
lat = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
}else{
lng2 = (d['I'].toDouble()*10000000000).longValue()
lat2 = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng = (gmap.get("lng")*10000000000).longValue()
lat = (gmap.get("lat")*10000000000).longValue()
}
}
def udata = new BasicDBObject()
udata.put("region", d['E'])
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
d['K'] = "修正区域"
}
}
}
println index
},0)
}
@@ -236,7 +314,7 @@ class LocationTest {
}
}
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
if(baiduRest){
bdlocationMap = getMapLocation(dist, baiduRest);
if(bdlocationMap){
@@ -280,7 +358,7 @@ class LocationTest {
region = r.district
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 10
}else {
score = 5
@@ -303,7 +381,7 @@ class LocationTest {
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 9
}else {
score = 5
@@ -320,7 +398,7 @@ class LocationTest {
region = r.district
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 8
}else {
score = 5
@@ -338,7 +416,7 @@ class LocationTest {
region = r.district
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 7
}else {
score = 5
@@ -366,15 +444,85 @@ class LocationTest {
return flag;
}
def findRegionRange(){
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city","福州市")
q.put("city",city+"市")
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion("福州",d.region)
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
}
return regionMap
}
def run5(){
def results = getAllErrorDistricts()
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
final def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = mapLocation(map)
if(ret){
rets << ret
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误")
}
def mapLocation(map){
def dist = CommonDatautils.findDistrictById(map[1])
if(dist){
def loactionMap = matchMap(dist)
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
if (loactionMap) {
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if (dist.region == loactionMap.region) {
println dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
} else {
println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
}
}
return ret
}
return null
}
def getAllErrorDistricts(){
def sql = "select cityid,commid from bigdata_community_distcheck where distname<>checkdist or checkdist is null";
def rets = []
sqlInstance.eachRow(sql,{r->
rets << [r.cityid, r.commid]
});
return rets
}
}
@@ -0,0 +1,27 @@
package com.fangjia.script.gj.location
class Point {
Double lat;
Double lng;
public Point(Double lng, Double lat){
this.lat = lat
this.lng = lng
}
Double getLat() {
return lat
}
void setLat(Double lat) {
this.lat = lat
}
Double getLng() {
return lng
}
void setLng(Double lng) {
this.lng = lng
}
}
@@ -0,0 +1,143 @@
package com.fangjia.script.gj.location
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.awt.geom.Point2D
class PointUtil {
public static final DBCollection BUSINESS_RANGE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border", "mongo.db.fangjia.com")
/**
* 判断当前位置是否在多边形区域内
*
* @param checkPoint 当前经纬度
* @param areaAround 围栏范围
* @return
*/
public static boolean checkIsInPolygon(Point checkPoint, List<Point> areaAround) {
Point2D.Double point = new Point2D.Double(checkPoint.getLat(), checkPoint.getLng());
List<Point2D.Double> pointList = new ArrayList<>();
for (Point everyPoint : areaAround) {
Point2D.Double polygonPoint = new Point2D.Double(everyPoint.getLat(), everyPoint.getLng());
pointList.add(polygonPoint);
}
return checkIsPtInPoly(point, pointList);
}
/**
* 判断点是否在多边形内,如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
*
* @param point 检测点
* @param pts 多边形的顶点
* @return 点在多边形内返回true, 否则返回false
*/
public static boolean checkIsPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
int N = pts.size();
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
boolean boundOrVertex = true;
//cross points count of x
int intersectCount = 0;
//浮点类型计算时候与0比较时候的容差
double precision = 2e-10;
//neighbour bound vertices
Point2D.Double p1, p2;
//当前点
Point2D.Double p = point;
//left vertex
p1 = pts.get(0);
//check all rays
for (int i = 1; i <= N; ++i) {
if (p.equals(p1)) {
//p is an vertex
return boundOrVertex;
}
//right vertex
p2 = pts.get(i % N);
//ray is outside of our interests
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
p1 = p2;
//next ray left point
continue;
}
//ray is crossing over by the algorithm (common part of)
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
//x is before of ray
if (p.y <= Math.max(p1.y, p2.y)) {
//overlies on a horizontal ray
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
return boundOrVertex;
}
//ray is vertical
if (p1.y == p2.y) {
//overlies on a vertical ray
if (p1.y == p.y) {
return boundOrVertex;
//before ray
} else {
++intersectCount;
}
} else {
//cross point on the left side
//cross point of y
double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
//overlies on a ray
if (Math.abs(p.y - xinters) < precision) {
return boundOrVertex;
}
//before ray
if (p.y < xinters) {
++intersectCount;
}
}
}
} else {
//special case when ray is crossing through the vertex
//p crossing over p2
if (p.x == p2.x && p.y <= p2.y) {
//next vertex
Point2D.Double p3 = pts.get((i + 1) % N);
//p.x lies between p1.x & p3.x
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
++intersectCount;
} else {
intersectCount += 2;
}
}
}
//next ray left point
p1 = p2;
}
//偶数在多边形外
if (intersectCount % 2 == 0) {
return false;
} else {
//奇数在多边形内
return true;
}
}
public static void main(def a){
new PointUtil().run()
}
def run(){
def q = new BasicDBObject()
q.put("name", "淞南")
def obj = BUSINESS_RANGE_COLL.findOne(q)
if(obj){
def pointStr = obj.border
if(pointStr){
def pointArrs = pointStr.split(";")
def list = []
for(def s : pointArrs){
def arr = s.split(",")
list << new Point(arr[0].toDouble(), arr[1].toDouble())
}
println checkIsInPolygon(new Point(121.491478,31.327947), list)
}
}
}
}
@@ -521,6 +521,10 @@ class GaoDeUtils {
return KeyList.get(count);
}
static void main(def a){
println search("如东县大豫镇豫东村委会","南通")
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.price
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
@@ -11,32 +12,35 @@ class EvaluationBatch {
task.run()
}
def run(){
def filePath = "D:\\数据\\农行\\东莞农行测试数据(御景花园18条估值)(人工询价).xlsx"
def filePath = "D:\\数据\\银联\\房价网测试数据.xlsx"
def city = "¶«Ý¸"
ExcelUtils.modify(filePath, { data ->
if(!data['M']){
// def address = data['C'].replace("\"","").trim()
// def rMap = SeaweedDataMatchProcess.doMatch(city, data['K'],data['L'], address );
def rMap = SeaweedDataMatchProcess.doMatch(city, data['D'],data['E'], null);
if(rMap){
def json = evaluation(rMap, data['H'], 7, data['I'],null)
def region = RegionData.formatRegion(data['A'], data['B'])
if(region){
def rMap = SeaweedDataMatchProcess.doMatch(data['A'], region, data['C'], null);
if(rMap){
def json = evaluation(rMap, data['D'], null, null,null)
// data['J'] = rMap.city
// data['K'] = rMap.region
// data['L'] = rMap.name
if(json){
def avgPrice = json.avg_price
def totalPrice = json.total_price
data['N'] = totalPrice
data['M'] = avgPrice
data['O'] = json.credit
data['P'] = rMap.region
data['Q'] = rMap.name
data['E'] = rMap.city
data['F'] = rMap.region
data['G'] = rMap.name
if(json){
def avgPrice = json.avg_price
def totalPrice = json.total_price
data['H'] = totalPrice
data['I'] = avgPrice
data['J'] = json.credit
// data['P'] = rMap.region
// data['Q'] = rMap.name
}
}
}
}
}, 0);
@@ -0,0 +1,44 @@
package com.fangjia.script.gj.price
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class NoPriceCalcutePrice {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new NoPriceCalcutePrice()
task.run()
}
def run(){
def citys = CcbCommonUtils.citys;
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
def map = getDistrictByCity(city, "2021-08-01")
for(def d : districts){
if(!map.containsKey(d.region+"@"+d.name)){
FangjiaClientUtils.sendDistrictPriceCalculate(d.city, d.region, d.name, 12);
println d.city+" "+d.region+" "+d.name
}
}
}
}
def getDistrictByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
def dbcursor = DIST_SELL_STAT.find(q)
def map = [:]
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
}
@@ -0,0 +1,65 @@
package com.fangjia.script.gj.region
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class MapAreaUpdate {
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
static main(args) {
def task = new MapAreaUpdate()
task.run()
}
def run(){
def rets = findAllCity()
for(def city : rets){
def retMap = findCityAndProvCode(city)
if(retMap){
update(retMap)
}
}
}
def update(retMap){
def q = new BasicDBObject()
q.put("fangjia_city",retMap.fangjia_city)
q.put("district", "ÊÐÏ½Çø")
def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("city_code", retMap.city_code)
udata.put("province_code", retMap.province_code)
FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false)
}
}
def findCityAndProvCode(def city){
def q = new BasicDBObject()
q.put("fangjia_city",city)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def map = [:]
for(def d : dbcursor){
if(d.city_code && d.province_code && d.district_code){
map.put("fangjia_city",d.fangjia_city)
map.put("city_code",d.city_code)
map.put("province_code",d.province_code)
break
}
}
return map;
}
def findAllCity(){
def q = new BasicDBObject()
q.put("fangjia_district",null)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def citys = []
for(def d : dbcursor){
citys << d.fangjia_city
}
citys.unique()
return citys
}
}
@@ -90,7 +90,7 @@ class AddressStandard {
}
}
}
},0)
},0, "Sheet1")
}
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
def findCrawlData(){
def q = new BasicDBObject()
// q.put("city","¸£ÖÝ")
q.put("source", "°²¾Ó¿Í")
q.put("city","福州")
q.put("source", "城市房产")
q.put("s_date",202012)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = "anjuke.com/"
def prefix = ".cityhouse.cn/ha/"
def keyList =new HashSet()
for(def db : dbcursor){
def url = null
@@ -77,11 +77,11 @@ class CrawlDistrictIdSelect {
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com"))
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
@@ -92,7 +92,7 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.root = "cityhouse.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
@@ -0,0 +1,45 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class MatchDistrict {
static main(args) {
def task = new MatchDistrict()
task.run()
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx"
def city = "深圳"
ExcelUtils.multiThreadingModify(filePath, { data ->
if(data['A']){
def region = RegionData.formatRegion(city, data['A'])
if(region){
def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null);
if(rMap){
data['E'] = rMap.city
data['F'] = rMap.region
data['G'] = rMap.name
}
}
}
}, 0);
}
}
//{
// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"),
// "guiding_type" : NumberInt(0),
// "estate_type" : "普通住宅",
// "house_type" : NumberInt(0),
// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"),
// "guiding_person" : "深圳房管局",
// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"),
// "category" : "district",
// "guiding_value" : NumberInt(48300),
// "name" : "中粮云景国际",
// "city" : "深圳",
// "region" : "光明"
//}
@@ -1,7 +1,57 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import groovy.sql.Sql
import java.text.SimpleDateFormat
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date(1617211269868);
print dateFormat.format(date);
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
//print dateFormat.format(date);
//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
//def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
//ExcelUtils.modify(path,{d->
// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
// d['G'] = districts.size()
//
// def q = new BasicDBObject()
// q.put("city", d['B'])
// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
// def obj = CITY_INFO_CONN.findOne(q);
// if(obj){
// d['H'] = obj.price_coverage*100
// }
//})
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
def countMap = [:]
onLineSqlInstance.eachRow(sql,{r->
def addTime = r.visit_time
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
def ips = countMap.get(month)
if(!ips){
ips = new HashSet();
countMap.put(month, ips)
}
ips << r.ip
});
countMap.each {k,v->
println k+" "+v.size()
}
@@ -47,12 +47,22 @@ class SFTP {
* @param sftp
*/
public void upload(String directory, String uploadFile, ChannelSftp sftp) {
FileInputStream stream = null
try {
sftp.cd(directory);
File file=new File(uploadFile);
sftp.put(new FileInputStream(file), file.getName());
stream = new FileInputStream(file)
sftp.put(stream, file.getName());
} catch (Exception e) {
e.printStackTrace();
} finally {
if(stream!=null){
stream.close();
}
// if (sftp != null) {
// sftp.disconnect();
// sftp.exit();
// }
}
}
@@ -35,17 +35,21 @@ class InsidePolygon {
public static List<Point> createPoint2(String points){
String[] array = points.split(";");
List<Point> list = new ArrayList<Point>();
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
if(points){
String[] array = points.split(";");
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
}
}
return list;
}
@@ -0,0 +1,75 @@
package com.fangjia.script.temp.shenjian
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
/**
* 快照监控
*
* @author shenjian
* @since 2021/8/27
*/
@BatchOwner(Watcher.ShenJian)
class SnapshotMonitor extends BaseBatch {
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
static void main(String[] args) {
runBatch(new SnapshotMonitor(), args)
}
@Override
void process(Object args) {
// /_snapshot/es_backup/weekly-snap-2021.05*
// 监控两个月的快照
String[] months = [DateUtils.formatDateByPattern(new Date(), "yyyy.MM"), DateUtils.formatDateByPattern(DateUtil.addMonthToDate(new Date(), -1), "yyyy.MM")]
String urlPrefix = "http://192.168.0.206:9200/_snapshot/es_backup/weekly-snap-"
String msg = "";
for (String month : months) {
// 必须分开写,不能有空格
String[] cmds = ["curl", "-XGET", urlPrefix + month + "*", "--user", "elastic:goojia123456"]
String result = execCurl(cmds)
JSONObject jsonObject = JSONObject.parse(result);
if (jsonObject.get("snapshots") != null) {
com.alibaba.fastjson.JSONArray jsonArray = jsonObject.get("snapshots")
for (JSONObject obj : jsonArray) {
String state = obj.get("state")
if (!"SUCCESS".equals(state)) {
msg = "【ES集群快照异常】\n" + obj.get("failures").toString()
DingDingMessageUtil.sendTextMessage(access_token, msg)
return ;
}
}
}
}
}
static String execCurl(String[] cmds) {
ProcessBuilder process = new ProcessBuilder(cmds);
Process p;
try {
p = process.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
StringBuilder builder = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null) {
builder.append(line);
builder.append(System.getProperty("line.separator"));
}
return builder.toString();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
}
@@ -0,0 +1,58 @@
package com.fangjia.script.test
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class RegionalismCodeStat {
DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018)
static main(args) {
def task = new RegionalismCodeStat()
task.run()
}
def run(){
def provinces = ["上海市","北京市","天津市","重庆市","福建省","贵州省","湖南省","广东省","湖北省","安徽省","四川省","江苏省","吉林省","河北省","河南省","海南省","江西省","云南省","甘肃省","山东省","陕西省","浙江省","青海省","辽宁省","黑龙江省","山西省","西藏自治区","新疆维吾尔自治区","宁夏回族自治区","内蒙古自治区","广西壮族自治区"]
// def provinces = ["宁夏回族自治区"]
provinces.each {v->
def sets = findCount(v)
println v +" "+ sets.size()
}
}
def findCount(v){
def q = new BasicDBObject()
q.put("province", v)
def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000)
def set = new HashSet()
for(def obj : dbcursor){
// if(obj.city=="省直辖县级行政区划" || obj.city=="自治区直辖县级行政区划"){
// //set << obj.region
// println "【"+obj.region+"】 过滤"
// continue
// }else{
// if(obj.region && !set.contains(obj.region)){
// set << obj.region
// println "【"+obj.region+"】 add"
// }
// }
if(obj.juweihui){
def k = obj.street+"##"+obj.juweihui
if(!set.contains(k)){
set << k
// println "【"+k+"】 add"
}
}
}
// set.each {k->
// println k
// }
return set;
}
}
@@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.CommonUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -59,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth()
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -81,7 +83,6 @@ class DataStandardBatch extends BaseBatch{
@Override
void process(Object args) {
def flag = true;
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
@@ -120,26 +121,35 @@ class DataStandardBatch extends BaseBatch{
void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
// if(dbo.avg_price){
if(dbo.name){
//标准化城市(包含region)
def data = standardCity(dbo);
if(data){
//匹配区域
data = standardRegion(data);
if(data){
//标准化小区名
def data3 = standardDistrict(data, dbo.avg_price);
if(data.city=="三亚" && data.region=="陵水"){
data.city = "陵水"
data.region = ""
def data3 = standardDistrictSimple(data, dbo.avg_price);
if(data3){
successCount++;
}
}else if(data){
data = standardRegion(data);
if(data){
//标准化小区名
def data3 = standardDistrict(data, dbo.avg_price);
if(data3){
successCount++;
}
}
}
}
// }else{
// saveLog(dbo, "无价格");
// updateStatus(dbo,statusMap.get("无价格"))
// }
}else{
saveLog(dbo, "无名称");
updateStatus(dbo,statusMap.get("无名称"))
}
count++;
}catch (Exception e){
@@ -253,6 +263,104 @@ class DataStandardBatch extends BaseBatch{
return data;
}
def standardDistrict2(def data, def avg_price){
def newName = NameProcess.nameStandard(data.city, data.region, data.name)
def district = null
if(newName){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address);
if(district){
}
}
if(district && avg_price){
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
saveSuccessLog(data, district, type);
updateStatus(data,statusMap.get("小区匹配成功"))
println district.city+" "+district.name+" success"
}else{
if(district){
// data.type = type
saveLog(data, "无价格");
updateStatus(data,statusMap.get("无价格"))
println data.city+" "+data.name+" fail(无价格)"
}else{
println "[匹配失败]"
saveLog(data, "小区匹配失败");
updateStatus(data,statusMap.get("小区匹配失败"))
println data.city+" "+data.name+" fail"
}
}
return district;
}
def standardDistrictSimple(def data, def avg_price){
def newName1 = null
def newName2 = null
if(data.name.contains("别名")){
newName1 = data.name.toString().substring(0, data.name.toString().indexOf("别名")-1)
newName2 = data.name.toString().substring(data.name.toString().indexOf("别名")+3)
}else{
newName1 = NameProcess.nameStandard(data.city, data.region, data.name)
}
def district = null
def type = null
if(newName1){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address);
if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){
district._id = data._id
type = 1
}else{
if(newName2){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address);
if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){
district._id = data._id
type = 1
}else{
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2)
if(score>0.8){
district._id = data._id
type = 11
}else{
district = null
}
}
}else{
if(district){
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1)
if(score>0.8){
district._id = data._id
type = 11
}else{
district = null
}
}
}
}
}
if(district && avg_price){
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
saveSuccessLog(data, district, type);
updateStatus(data,statusMap.get("小区匹配成功"))
println district.city+" "+district.name+" success"
}else{
if(district){
data.type = type
saveLog(data, "无价格");
updateStatus(data,statusMap.get("无价格"))
println data.city+" "+data.name+" fail(无价格)"
}else{
println "[匹配失败]"
saveLog(data, "小区匹配失败");
updateStatus(data,statusMap.get("小区匹配失败"))
println data.city+" "+data.name+" fail"
}
}
return district;
}
def standardDistrict(def data, def avg_price){
//名称匹配,名称别名匹配,地址别名匹配
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
@@ -368,30 +476,10 @@ class DataStandardBatch extends BaseBatch{
}
if(district){
if(data.region != district.region){
//type = 12;//区域不同
type = 12;//区域不同
district = null;
}else{
def score1 = 0;
if(data.address && district.address){
score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address);
}
if(score1 && score1 >=0.8){
type = 13;//地址相似度在0.8以上
}else{
if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){
type = 13;
}else{
if(!addr){
def openDistName = district.name.replace("小区","")
def name = data.name.replace("小区","")
if(openDistName.contains(name) || name.contains(openDistName)){
type = 15;//匹配接口通过名称配上,名称包含
}
}
}
//type = 11;
district = null;
}
type=15
}
}
}
@@ -495,10 +583,16 @@ class DataStandardBatch extends BaseBatch{
}*/
def getMatchNameFromMap(def city, def region, def disName, def address){
def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName);
dbo = addressCheck(dbo, region, address);
def addrFlag = null
if(address){
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
}
}
if(!dbo){
//别名
def distrs = findAliasNameFromMap(city, region, disName);
@@ -506,7 +600,7 @@ class DataStandardBatch extends BaseBatch{
dbo = distrs[0]
}else if(distrs && distrs.size()>1 && address){
for(def d : distrs){
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
dbo = d;
break;
}
@@ -570,10 +664,16 @@ class DataStandardBatch extends BaseBatch{
return dbo;
}*/
def getMatchAddressFromMap(def city, def region, def address, def name){
def dbo = findSameNameFromMap(city, region, address);
def dbo = null
if(address && isStandardRoad(address)){
dbo = findSameNameFromMap(city, region, address);
}
if(!dbo){
def result = findAddressFromMap(city, region, address);
if(result.size()==1){
def result = null
if(address && isStandardRoad(address)){
result = findAddressFromMap(city, region, address);
}
if(result && result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
def score = null;
@@ -621,6 +721,15 @@ class DataStandardBatch extends BaseBatch{
return dbo;
}
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
def addressCheck(def dbo, def region, def address){
if(dbo && !region && address && address.length()>3){
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){
@@ -673,7 +782,6 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("name", "银河湾花园(别墅)")
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -38,6 +38,152 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
def cityList = [
"合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319",
"安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","淮北":"huaibei|9357","滁州":"chuzhou|10266",
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
"福州":"fz|304","厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710",
"宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761",
"石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757",
"福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133",
"漳浦":"zhangpu|717",
"深圳":"sz|4","广州":"gz|3","东莞":"dg|413","佛山":"fs|222","中山":"zs|771",
"珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791",
"肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303",
"阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449",
"潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725",
"博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456",
"南宁":"nn|845","柳州":"liuzhou|7133","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046",
"北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340",
"来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774",
"北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119",
"贵阳":"gy|2015","遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506",
"毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628",
"清镇":"qingzhen|12703",
"兰州":"lz|952","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154",
"酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428",
"陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390",
"海口":"haikou|2053","三亚":"sanya|2422","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380",
"儋州":"danzhou|10394",
"郑州":"zz|342","洛阳":"luoyang|556","新乡":"xx|1016","南阳":"ny|592","许昌":"xc|977",
"平顶山":"pds|1005","安阳":"ay|1096","焦作":"jiaozuo|3266","商丘":"sq|1029","开封":"kaifeng|2342",
"濮阳":"puyang|2346","周口":"zk|933","信阳":"xy|8694","驻马店":"zmd|1067","漯河":"luohe|2347",
"三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123",
"禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010",
"项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101",
"沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393",
"通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219",
"舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322",
"沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185",
"范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603",
"哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
"佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862",
"伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721",
"肇州":"zhaozhou|382",
"武汉":"wh|158","宜昌":"yc|858","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
"黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296",
"咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517",
"仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896",
"武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484",
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
"长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914",
"湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
"祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473",
"邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532",
"湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095",
"石家庄":"sjz|241","保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572",
"秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993","张家口":"zjk|3328",
"承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048",
"正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577",
"雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657",
"沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395",
"固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066",
"沙河":"shaheshi|755",
"苏州":"su|5","南京":"nj|172","无锡":"wx|93","常州":"cz|463","徐州":"xz|471",
"南通":"nt|394","扬州":"yz|637","盐城":"yancheng|613","淮安":"ha|968","连云港":"lyg|2049",
"泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
"如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147",
"扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402",
"邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401",
"东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148",
"姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959",
"句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976",
"金湖":"jinhu|975","江阴":"jiangyins|34984",
"南昌":"nc|669","赣州":"ganzhou|2363","九江":"jj|2247","宜春":"yichun|5709","吉安":"ja|2364",
"上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115",
"鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118",
"丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137",
"浮梁":"fuliangxian|9071",
"长春":"cc|319","吉林":"jl|700","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315",
"白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171",
"梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708",
"梨树县":"lishu|10176",
"沈阳":"sy|188","大连":"dl|147","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722",
"营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038",
"本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306",
"瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733",
"银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421",
"呼和浩特":"hu|811","包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015",
"呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976",
"乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043",
"西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
"黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904",
"青岛":"qd|122","济南":"jn|265","烟台":"yt|228","潍坊":"wf|362","临沂":"linyi|505",
"淄博":"zb|385","济宁":"jining|450","泰安":"ta|686","聊城":"lc|882","威海":"weihai|518",
"枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","东营":"dy|623","菏泽":"heze|5632",
"滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146",
"寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690",
"高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235",
"蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967",
"新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884",
"茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640",
"无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739",
"乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888",
"阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334",
"汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238",
"宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373",
"临朐":"linqu|374","鄄城":"juancheng|5635",
"太原":"ty|740","临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854",
"长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453",
"朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354",
"泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227",
"西安":"xa|483","咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163",
"榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832",
"神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948",
"成都":"cd|102","绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380",
"自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928",
"遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803",
"凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817",
"甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706",
"射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148",
"乌鲁木齐":"xj|984","昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499",
"喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475",
"和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562",
"图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845",
"拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648",
"那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684",
"昆明":"km|541","曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040",
"丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409",
"德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432",
"怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533",
"杭州":"hz|79","宁波":"nb|135","温州":"wz|330","金华":"jh|531","嘉兴":"jx|497",
"台州":"tz|403","绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793",
"舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333",
"诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334",
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
"安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409",
"平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
"江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
]
static main(args) {
runBatch(new Esf58Crawl(), args)
}
@@ -45,18 +191,29 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
for(def i=0;i<5;i++){
def city = "¸£ÖÝ"
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
distMap.put(city, districts)
def citys = ["成都","南京","福州","厦门","泉州","东莞","广州","广州","广州","西安","沈阳","大连","昆明","太原","郑州","乌鲁木齐","烟台","绵阳","邯郸","潍坊","嘉兴","惠州","珠海","海口"]
cityList.each{city, str->
for(def i=0;i<1;i++){
// def city = "福州"
if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://fz.58.com/xiaoqu/");
addCrawlKeys(city);
addDetailContent(city, url);
Thread.sleep(1000*60*10)
// Thread.sleep(1000*60*10)
}
}
}
}
}
def addCrawlKeys(city){
def districts = distMap.get(city);
for(def i=0;i<districts.size();i++){
@@ -1,8 +1,10 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash
import org.bson.types.ObjectId
/**
* ¸£ÖÝÅ©ÉÌÐÐ
@@ -13,8 +15,9 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客"]
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -29,16 +32,19 @@ class SynchDistrictPrice {
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
}
}
def findCrawlDistrictDetails(city, root, source){
def curMonthFirstDay = DateUtils.getFirstDayOfMonth(new Date())
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
// q.put("name","兰州万达城")
q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay));
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
@@ -47,6 +53,7 @@ class SynchDistrictPrice {
keys.put("address", 1)
keys.put("avg_price", 1)
keys.put("root", 1)
keys.put("c_date", 1)
keys.put("url", 1)
def rets = []
def dbcursor = districtDetailColl.find(q, keys)
@@ -74,6 +81,7 @@ class SynchDistrictPrice {
q.put("city", city)
def count = thirdPartyColl.count(q)
def addIndex = 0
def updateIndex = 0
if(count==0){
try{
thirdPartyColl.insert(rets)
@@ -83,13 +91,27 @@ class SynchDistrictPrice {
println "["+source +"-"+city + "] add:"+ rets.size()+" ԭץȡ£º"+count
}else{
for(def r : rets){
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(!flag){
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(obj){
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
addIndex++
thirdPartyColl.save(r)
}else{
thirdPartyColl.save(r)
}
// if(!obj){
// addIndex++
// thirdPartyColl.save(r)
// }else{
// if(r.avg_price && !obj.avg_price){
// def udata = new BasicDBObject()
// udata.put("avg_price", r.avg_price)
// thirdPartyColl.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',udata))
// updateIndex++
// }
// }
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
}
@@ -102,10 +124,7 @@ class SynchDistrictPrice {
q.put("city", city)
q.put("region", region)
q.put("name", name)
def count = thirdPartyColl.count(q)
if(count){
return true
}
return false
def obj = thirdPartyColl.findOne(q)
return obj
}
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202105);
def count2 = findTotalCount(source, 202106);
def count1 = findTotalCount(source, 202108);
def count2 = findTotalCount(source, 202109);
println source +" "+count1 +" "+count2
}
}
@@ -41,8 +41,10 @@ class ThirdDataToGatherBatch extends BaseBatch{
@Override
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
def cityIndex = 0;
final CountDownLatch latch = new CountDownLatch(citys.size())
citys.each { city ->
@@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{
@Override
void process(Object args) {
citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""])
citys.addAll(["¸£ÖÝ"])
// getNoDataCitys();
def cityurl = "http://www.zhuge.com/areachange/"
def cityUrls = getCityUrlCrawal(cityurl);
@@ -531,8 +531,10 @@ public class DataUtil {
if(!(address ==~ /.*(½»»ã|½»¿Ú|½»²æ¿Ú)$/)){
address += "½»»ã"
}
}else{
address = null
}
address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
// address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
}
return address
}
@@ -921,13 +921,13 @@ class DateUtils {
public static String getRandomDate(def startDate, def endDate){
Date randomDate = randomDate(startDate, endDate);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
String result = format.format(randomDate);
return result;
}
static main(args){
println getRandomDate("2021-08-01","2021-08-31")
// nowFirstDay();
// nowLastDay();
/*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
@@ -955,8 +955,9 @@ class DateUtils {
// Date randomDate = randomDate("2010-09-20", "2011-09-22");
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
// String result = format.format(randomDate);
System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
// System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
println DateUtils.getDayOfWeek(new Date())
}
@@ -80,7 +80,7 @@ class ElasticUtils {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
@@ -60,7 +60,8 @@ public class ExcelUtils{
FileInputStream inp = null;
try {
inp = new FileInputStream(file);
Workbook workbook = WorkbookFactory.create(inp)
BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 200)
Workbook workbook = WorkbookFactory.create(bufferedInputStream)
workbook.setForceFormulaRecalculation(true)
Sheet sheet = workbook.getSheetAt(sheetIndex);
//ÐкÅ
@@ -82,6 +83,9 @@ public class ExcelUtils{
}
}
ret.add(rowData);
if (ret.size() % 10000 == 0) {
println(ret.size())
}
}
} catch (IOException e) {
e.printStackTrace();
@@ -140,6 +140,7 @@ public class OpenApiClient {
return null
}
public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) {
if (StringUtils.isBlank(input)) {
@@ -624,7 +625,7 @@ public class OpenApiClient {
params.put("token", TOKEN)
try {
JSONObject result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/evaluate", params)
if (result.getInt("code") == 200) {
if (result && result.getInt("code") == 200) {
if (result.containsKey("result")) {
// if (result.get("result").get("credit") == 0) {
// return null
@@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def city = "À³Îß"
def destCity = "¼ÃÄÏ"
def region = ""
def destRegion = "À³Îß"
def city = "쭤捺"
def destCity = "쭤捺"
def region = "섈적"
def destRegion = "촙쏠"
@@ -26,6 +26,7 @@ class GaoDeApiUtils {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("location", location)
params.put("extensions","all")
String content
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")