This commit is contained in:
2022-10-24 16:46:31 +08:00
94 changed files with 5224 additions and 2075 deletions
@@ -23,7 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_all;
def rows = []
def headers = []
for (def city : citys) {
@@ -69,7 +69,7 @@ class BigdataBuildingBatch extends BaseBatch{
q.put("city", city)
q.put("status", 1)
// q.put("name","辽源一村")
def dbcursor = LOU_DONG_STAT.find(q)
def dbcursor = LOU_DONG_STAT.find(q).batchSize(5000)
def map = [:]
def set = new HashSet()
for(def db : dbcursor){
@@ -28,7 +28,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
citys = citys.unique()
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -35,7 +35,8 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_92;
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -43,18 +44,19 @@ class BigdataCommunityBatch extends BaseBatch{
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" 小区量:"+dists.size()
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println city +" "+rets.size()
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
@@ -220,7 +222,7 @@ class BigdataCommunityBatch extends BaseBatch{
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name
println dist.city + " "+dist.name+" error"
e.printStackTrace()
}
@@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
@@ -32,8 +33,11 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
// def startDate = 202206
// def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -52,49 +57,52 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = ['南京', '无锡', '徐州', '常州', '苏州', '南通', '连云港', '淮安', '盐城', '扬州', '镇江', '泰州', '宿迁']//CcbCommonUtils.citys;
// def ccbCitys = CcbCommonUtils.citys;
for(def city : citys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for(def city : citys){
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
//
// def trades = getCityData(city, statMap, ppeDists)
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成"
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true);
println "txt 生成 " + rows.size()
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -102,6 +110,9 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
// def lianjiaPriceMap = findThirdPartyPrice("上海")
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
@@ -109,7 +120,8 @@ class BigdataDealBatch extends BaseBatch{
try {
totalCount.addAndGet(1)
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, lianjiaPriceMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
@@ -135,147 +147,176 @@ class BigdataDealBatch extends BaseBatch{
return result
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
q.put("city", city)
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for (def db : dbcursor) {
def price = null
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
price = db.lianjia_esf_price
}
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
price = db.beike_esf_price
}
if(price){
map.put(db.region+"##"+db.name, price)
}
}
return map
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){
def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
def price = lianjiaPriceMap.get(obj.region+"##"+obj.name)
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
@@ -289,7 +330,7 @@ class BigdataDealBatch extends BaseBatch{
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
ret << CcbCommonUtils.getCcbCity(trade.city)
if(data){
ret << data.district_code
ret << data.district
@@ -543,11 +584,14 @@ class BigdataDealBatch extends BaseBatch{
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
q.put("weekend", new BasicDBObject('$gte', weekend))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
def price = map.get(d.region+"##"+d.district_name)
if(!price && d.value_type==2){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
}
return map
}
@@ -572,7 +616,7 @@ class BigdataDealBatch extends BaseBatch{
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
final CountDownLatch latch = new CountDownLatch(trades.size())
@@ -10,6 +10,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
/**
* 建行模型估值项目
* 区域数据同步
@@ -17,33 +19,74 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class BigdataDistrictBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new BigdataDistrictBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def datas = findDistrictCodeValue(city)
if(datas){
rows.addAll(datas)
}
println city+" "+ datas.size()
}
def allDatas = createDatas(rows)
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
def datas = readText("D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.txt")
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
def index = 0
for(def line : datas){
def arrs = line[0].split("\\|@\\|")
def sql = "select count(*) as count from BIGDATA_DISTRICT_GJ where distid='${arrs[0]}'";
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
if(count==0){
rets << line
}
}
writeTxt(rets, "D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ_add.txt")
// def citys = CcbCommonUtils.citys_all;
// def rows = []
// for(def city : citys){
// def datas = findDistrictCodeValue(city)
// if(datas){
// rows.addAll(datas)
// }
// println city+" "+ datas.size()
// }
//
// def allDatas = createDatas(rows)
//
// def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
//
// def rets = []
// rets << headers
// rets.addAll(allDatas)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
// FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
}
def writeTxt(def rows, def path){
StringBuilder builder = new StringBuilder()
for(def row : rows){
builder.append(row)
builder.append('\n')
}
org.apache.commons.io.FileUtils.writeStringToFile(new File(path), builder.toString(), "GBK")
}
private def readText(def fileName){
File file = new File(fileName)
def ret = [];
//对文件内容的每一行进行分割处理,比较常用在处理csv文件
file.splitEachLine(" ") {
def temp = it
ret << temp;
}
//转为Writable对象,可重定向输出
file.asWritable()
return ret;
}
def createDatas(rows){
def allRows = []
def index = 1
@@ -72,16 +115,16 @@ class BigdataDistrictBatch extends BaseBatch{
def findDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
// q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
q.put("fjw_city", city)
// def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
def dbcursor = CcbCommonUtils.FANGJIA_AREA_CODE.find(q)
def lists = []
for(def db : dbcursor){
if(db.district_code){
def m = [:]
m.id = db._id.toString()
m.dist_name = db.district
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fjw_district, "")
m.city_id = db.city_code
m.dist_code = db.district_code
lists << m
@@ -67,20 +67,39 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
def index = 0
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
statCity(citys)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220618, 20220719);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
}
def statCity(def citys){
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
def houseCitys = []
for(def d : datas){
houseCitys << d['C']
}
for(def city : citys){
if(!houseCitys.contains(city)){
def scity = CcbCommonUtils.specialCityMap.get(city)
if(!scity){
println city
}
}
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
@@ -538,7 +557,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
rowMap.houseId = seawater.id
rowMap.city = city
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
@@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{
@Override
public void process(Object args) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def citys = CcbCommonUtils.citys_252
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1)
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1)
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1)
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
def rets = []
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows1)
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
// rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
// rets.addAll(rows3)
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
////
// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
// rets.addAll(rows4)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
// rets.addAll(rows5)
//
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
if(CommonDatautils.isWindow()){
@@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['F'].contains("中学")){
if(d['B'].contains("中学")){
ret << "ed138"
}else if(d['F'].contains("小学")){
}else if(d['B'].contains("小学")){
ret << "ed114"
}
}
@@ -16,12 +16,16 @@ import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// public static def citys = ["武汉"]
public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com")
// public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com")
public static final DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
@@ -34,39 +38,76 @@ class CcbCommonUtils {
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"]
public static DSCollection dataStoreConnection = null;
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def getRunningCitys(def args){
def citys = []
println "args 参数:"+args.toString()
if(args){
def countStr = args[0]
if(countStr=="344"){
citys = CcbCommonUtils.citys_all
}else if(countStr=="92"){
citys = CcbCommonUtils.citys_92
}else if(countStr=="252"){
citys = CcbCommonUtils.citys_252
}
}else{
citys = CcbCommonUtils.citys_92
}
println "运行城市数量:"+citys.size()
return citys;
}
public static def findCityCodeValue(){
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
def cityCodeMap = [:]
FANGJIA_AREA_CODE.find().each {it->
def code = cityCodeMap.get(it.fjw_city)
if(!code){
cityCodeMap.put(it.fjw_city, it.city_code.toString())
}
}
// def oldCityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
return cityCodeMap;
}
public static def getCcbCity(def city){
if(city && specialCityMap.containsKey(city)){
return specialCityMap.get(city)
}
return city;
}
public static def findAllDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
q.put("fjw_city", city)
//q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
map.put(db.fangjia_district, db)
map.put(db.fjw_district, db)
}
return map;
}
public static def findAllCityDistrictCodeValue(city){
def q = new BasicDBObject()
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
def distMap = map.get(db.fangjia_city)
def distMap = map.get(db.fjw_city)
if(!distMap){
distMap = [:]
map.put(db.fangjia_city, distMap)
map.put(db.fjw_city, distMap)
}
distMap.put(db.fangjia_district, db)
distMap.put(db.fjw_district, db)
}
return map;
}
@@ -77,8 +118,16 @@ class CcbCommonUtils {
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def rets = []
def sets = new HashSet()
for(def d : dbcursor){
rets << d
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
if(!sets.contains(key)){
rets << d
sets << key
}else{
println key
}
}
return rets
}
@@ -368,7 +417,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","辽源一村")
// q.put("name","云山尚城")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils
class CcbFtpUtils {
static def ftpServer = "192.168.0.45"
static def ftpServer = "192.168.0.45"//192.168.0.45
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
@@ -370,24 +370,24 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202206
def citys = CcbCommonUtils.citys_92
def month = 202209
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202207)
def rets = findPeGteThirdParty(city, 202210)
def total = 0
println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
def cityId = cityCodeDatas.get(city)
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
@@ -62,7 +62,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code
ret << areaData.district
ret << ps.avg_price
@@ -15,31 +15,39 @@ class ExportOracleData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def citys = null
static main(args) {
def task = new ExportOracleData()
task.run()
task.run(args)
}
def run() {
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
//不会变
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
exportBigdataPoiData()//
exportBigdataDistrictData()//
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
exportGuidedPrice()//随人工改变//
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
// //处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
exportBigdataCommunity()//
exportCommunityStaging()//没cityid
exportCommunityBuileType()//
exportThirdPartyPrice2(lastMonth, new_dists)
exportBigdataForsale("2022-06-19", new_dists)
exportBigdataDeal("2022-06-01", new_dists)
exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//第三方网站权重表
exportWebsiteRate();
//计算结果相关
@@ -93,26 +101,61 @@ class ExportOracleData {
}
def exportBigdataPoiData() {
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj";
def exportWebsiteRate(){
def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.DISTID
data << r.ID
data << r.POI_NAME
data << r.DATA_SOURCE_ID
data << r.DATA_SOURCE
data << r.DATA_SOURCE_MAP
data << r.WEIGHT
data << r.TYPE
data << r.TYPE_CODE
data << r.LATITUDE
data << r.LONGITUDE
data << r.REMARK
data << r.SOURCECODE
data << r.CITYID
data << r.CITYNAME
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "上传Ftp完成......"
}
def exportBigdataPoiData() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.DISTID
data << r.ID
data << r.POI_NAME
data << r.TYPE
data << r.TYPE_CODE
data << r.LATITUDE
data << r.LONGITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
@@ -128,8 +171,9 @@ class ExportOracleData {
}
def exportCommunityStaging() {
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
def allRows = []
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
@@ -158,9 +202,9 @@ class ExportOracleData {
// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def allRows = []
// for(def city : citys){
// def cityid = cityCodeDataMap.get(city)
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj";
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
@@ -173,7 +217,8 @@ class ExportOracleData {
data << r.WORKDATE
allRows << data
});
// }
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
@@ -189,28 +234,31 @@ class ExportOracleData {
}
def exportBigdataDistrictData() {
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.LJXZQH
data << r.LJXZQHDM
data << r.XZQHDM
data << r.WORKDATE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.LJXZQH
data << r.LJXZQHDM
data << r.XZQHDM
data << r.WORKDATE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -227,26 +275,30 @@ class ExportOracleData {
def exportGuidedPrice() {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.DEALDATE
data << r.AVG_PRICE
data << r.GUIDING_PERSON
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.DEALDATE
data << r.AVG_PRICE
data << r.GUIDING_PERSON
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -261,25 +313,29 @@ class ExportOracleData {
}
def exportCommunityGuidedPrice(def date) {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.AVGPRICE
data << r.SOURCETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.AVGPRICE
data << r.SOURCETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -296,30 +352,33 @@ class ExportOracleData {
def exportBigdataDeal(def dealDate, def new_dists) {
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}"
def rets1 = dealData(sql1)
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where distid in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
allRows = dealData(sql)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
allRows = dealData(sql)
}
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
@@ -328,8 +387,6 @@ class ExportOracleData {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
@@ -462,25 +519,6 @@ class ExportOracleData {
def exportThirdPartyPrice2(price_date, new_dists) {
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"price_date=${price_date} and distid not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"distid in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} ";
allRows = getThirdPartyDatas(sql)
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -488,9 +526,37 @@ class ExportOracleData {
// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"CITYID='${cityCode}' and distid in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} ";
allRows = getThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
index = index + 1
if(index==1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
}
@@ -517,53 +583,58 @@ class ExportOracleData {
}
def exportBigdataCommunity(){
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100)
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100)
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
@@ -572,18 +643,15 @@ class ExportOracleData {
def exportBigdataForsale(publishDate, new_dists){
//["北京","上海"]
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists){
@@ -601,7 +669,13 @@ class ExportOracleData {
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
@@ -743,8 +817,8 @@ class ExportOracleData {
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
@@ -958,55 +1032,55 @@ class ExportOracleData {
//
// for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
@@ -0,0 +1,267 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class FjwPricePkCcbPrice {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
// task.addFjwPrice()
// task.addCcbPrice()
task.addMarketPrice()
}
def addMarketPrice(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\拍卖南京.xlsx",{d->
if(d['D']) {
d['K'] = d['D']
}else if(d['H']){
d['K'] = d['H']
}
})
}
def addCcbPrice(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d->
if(d['B']){
def cityId = cityCodeDataMap.get(d['B'])
def dist = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
if(dist){
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209)
if(map){
d['I'] = map.avg_price
d['J'] = map.calc_type
d['k'] = map.credit
}
}
// def dist = CommonDatautils.findDistrictById(d['B'])
// if(dist){
// d['J'] = dist.status
// }
}
})
}
def updatePrice(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d->
index++
println index
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
if(dbo){
d['G'] = dbo.future_value
}
})
}
def addFjwPrice(){
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
if(!d['M']){
def dist = CommonDatautils.findDistrictById(d['D'])
if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
if(dbo){
d['M'] = dbo.avg_price
}
}
}
})
}
def run(){
def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size()
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
println "fjw总量:"+ccbPrices.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(ccbPrices.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<ccbPrices.size();i++){
final def rMap = ccbPrices[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def row = []
def dist = CommonDatautils.findDistrictById(rMap.commid)
if(dist){
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
def fjwPrice = null
def fjwCalPrice = null
if(priceMap){
fjwPrice = priceMap.avg_price
fjwCalPrice = priceMap.calc_value
}
row << rMap.commid
row << dist.city
row << dist.region
row << dist.name
row << fjwPrice
row << fjwCalPrice
if(priceMap && priceMap.value_type){
if(priceMap.value_type == 1){
row << "指导价"
}else if(priceMap.value_type == 2){
row << "自身样本"
}else if(priceMap.value_type == 3){
row << "第三方"
}else if(priceMap.value_type == 4){
row << "交易数据"
}else if(priceMap.value_type == 5){
row << "周边楼盘"
}else if(priceMap.value_type == 6){
row << "推导"
}else if(priceMap.value_type == 9){
row << "神经网络"
}else if(priceMap.value_type == 10){
row << "无方式"
}else{
row << ""
}
}else{
row << ""
}
row << rMap.avg_price
row << rMap.calc_type
if(fjwPrice && rMap.avg_price){
def precent = (fjwPrice - rMap.avg_price)*100/rMap.avg_price
row << DataHandleUtils.getDoubleRound(2, precent)
}else{
row << ""
}
rets << row
}else{
println rMap.commId + " 没找到小区"
}
println totalCount.get()+" "+ rMap.cityname +" "+rMap.commname
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await();
def headers = ["小区ID","城市","区域","小区名","房价网价格","房价网预期价格","房价网计算方式","建行价格","建行计算方式","价格差比(房价网价格-建行价格)*100/建行价格"]
batchCreateFile(rets, "D:\\数据\\建行\\测试\\房价网与建行均价比对.csv",headers,"GBK")
}
def getCommunityBasePriceGj(yearMonth) {
def sql = "select CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE from community_base_price_gj where year_month=${yearMonth} order by cityid";
def allRows = []
sqlInstance236.eachRow(sql, { r ->
def map = [:]
map.cityname = r.CITYNAME
map.distname = r.DISTNAME
map.commid = r.COMMID
map.commname = r.COMMNAME
map.year_month = r.YEAR_MONTH
map.calc_avg_price = r.CALC_AVG_PRICE
map.avg_price = r.AVG_PRICE
// 计算类型,1:人工估值 2:样本计算 3:神经网络
if(r.CALC_TYPE){
if(r.CALC_TYPE == 1){
map.calc_type = "人工估值"
}else if(r.CALC_TYPE == 2){
map.calc_type = "样本计算"
}else if(r.CALC_TYPE == 3){
map.calc_type = "神经网络"
}
}
allRows << map
});
return allRows
}
def getCommunityBasePriceGjByCommid(cityid, commid, yearMonth) {
def sql = "select AVG_PRICE, CALC_TYPE,CONFIDENCE from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} and commid='${commid}'";
def map = [:]
sqlInstance236.eachRow(sql, { r ->
map.avg_price = r.AVG_PRICE
// 计算类型,1:人工估值 2:样本计算 3:神经网络
if(r.CALC_TYPE){
if(r.CALC_TYPE == 1){
map.calc_type = "人工估值"
}else if(r.CALC_TYPE == 2){
map.calc_type = "样本计算"
}else if(r.CALC_TYPE == 3){
map.calc_type = "神经网络"
}
}
map.credit = r.CONFIDENCE
});
return map
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "@|@"))
sb.append(System.getProperty("line.separator"));
println index
if(index%10000==0){
println index+"开始写入CSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"结束写入CSV "+new Date().toLocaleString()
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_252;
def rows = []
for (def city : citys) {
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-06-1", "2022-07-21")
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -147,7 +147,7 @@ class GuiDingPriceBatch extends BaseBatch{
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
@@ -24,7 +24,7 @@ class StagingDistrictBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
File diff suppressed because it is too large Load Diff
@@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202206//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202204
def endDate = 202206
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202107
def endDate = 202109
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
@@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
}
@@ -73,15 +73,15 @@ class TradeStatLog {
}
def run(){
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_92;
def rets = []
rets << ["城市","4月总量","5月总量","6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-04-01"
def date = "2022-05-01"
def endDate = "2022-06-01"
def newDate = "2022-07-01"
def startDate = "2022-07-01"
def date = "2022-08-01"
def endDate = "2022-09-01"
def newDate = "2022-10-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-06成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\${newDate}成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -0,0 +1,26 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\建行最终区域编码.xlsx", 0, 1)
for(def d : datas){
def dbo = new BasicDBObject()
dbo.put("province",d['C'])
dbo.put("city", d['D'])
dbo.put("district", d['G'])
dbo.put("city_code", d['E'].toInteger())
dbo.put("district_code",d['F'].toInteger())
dbo.put("fjw_city", d['A'])
dbo.put("fjw_district", d['B'])
dbo.put("c_date", new Date())
FANGJIA_AREA_CODE.save(dbo)
}
@@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_all;
def rows = []
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for (def city : citys) {
@@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// })
// }
// addBigdataDistrictFJW()
addBigdataDistrictFJW()
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
def index = 0
for (def city : citys) {
index++
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
//
// for(int i=0;i<10;i++){
// if(districts.size()==0){
// Thread.sleep(1000*60)
// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
// }else{
// break
// def index = 0
// for (def city : citys) {
// index++
// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
////
//// for(int i=0;i<10;i++){
//// if(districts.size()==0){
//// Thread.sleep(1000*60)
//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
//// }else{
//// break
//// }
//// }
//
// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
//
// for(def dist : districts){
// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
// if(obj){
// if(obj.alias && obj.alias.size()>0){
// dist.commalias = obj.alias.get(0)
// }else{
// dist.commalias = ""
// }
// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
// dist.houseuse = "11"
// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
// dist.compid = new ObjectId() //物业公司id
// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
// }
// }
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
for(def dist : districts){
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
if(obj){
if(obj.alias && obj.alias.size()>0){
dist.commalias = obj.alias.get(0)
}else{
dist.commalias = ""
}
dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
dist.houseuse = "11"
dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
dist.compid = new ObjectId() //物业公司id
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
}
}
println index +" "+city +" 小区加载完成 "+districts.size()
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
println index +" "+city +" 1.小区表完成"
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 2.小区建筑结构表"
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 3.小区房屋类型表"
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 4.小区建筑年代表"
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 5.小区费用表"
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 6.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
println index +" "+ city + " 7.地理位置表"
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
}
// println index +" "+city +" 小区加载完成 "+districts.size()
// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 1.小区表完成"
// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 2.小区建筑结构表"
// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 3.小区房屋类型表"
// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 4.小区建筑年代表"
// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 5.小区费用表"
// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 6.小区房屋用途表"
// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
// println index +" "+ city + " 7.地理位置表"
//
//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
//
// }
}
def addBigdataCommunityGJRH(def districts, def cityid){
@@ -557,14 +557,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){
def rets = []
def num = 0
for(def d : districts) {
num = num + 1
def ret = []
def detailMap = citySeqnoMap.get(d.cityid)
def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap);
ret << d.cityid
ret << "5"
ret << d.commid
ret << "1"
ret << seqno
if(detailMap && seqno){
ret << seqno
}else{
ret << num
}
ret << d.lng
ret << d.lat
ret << ""
@@ -22,25 +22,29 @@ class ExportDistrictRongHe extends BaseBatch{
runBatch(new ExportDistrictRongHe(), args);
}
def citys = []
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()
exportBigdataBuildingFJW()
exportBigdataCommfeeGj()
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()
exportBigdataCommhousetypeGJ()
exportBigdataCommbuildyearGj()
exportBigdataCommhouseuseGJ()
exportBigdataPositionGJ()
exportBigdataDistrictFJW()
exportBigdataAliasFJW()
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
@@ -59,14 +63,14 @@ class ExportDistrictRongHe extends BaseBatch{
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//
//
//// //检查ok文件是否上传
//// //检查ok文件是否上传
//// exportAllBigdataDealFJW()
//// for(def city : citys) {
//// def cityCode = cityCodeDataMap.get(city)
////
// //按天导(一个文件夹下)
// //按天导(一个文件夹下)
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
// //按天导(每天一个文件夹)
// //按天导(每天一个文件夹)
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
//
//// exportBigdataCommunityGJRH(cityCode)
@@ -94,261 +98,308 @@ class ExportDistrictRongHe extends BaseBatch{
// exportOk()
// }
def exportBigdataCommunityGJRH(cityCode){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << ''
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
def index = 0
for(def city : citys){
def allRows = []
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << ''
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
if(allRows && allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
}
def exportBigdataCommhousetypeGJ(cityCode){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
def exportBigdataCommhousetypeGJ(){
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildyearGj(){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
}
def exportBigdataCommfeeGj(){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
}
def exportBigdataCommhouseuseGJ(){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
}
def exportBigdataPositionGJ(){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
}
def exportBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
}
def exportBigdataAliasFJW(){
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.city_id
data << r.ca_id
data << r.address_id
data << r.address_name
data << r.status
data << r.exp_dt
data << r.version
data << r.is_key_name
data << r.data_source
data << r.description
data << r.alias_grade
data << r.addr_type_cd
data << r.ifexamine
data << r.ifdisplay
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.city_id
data << r.ca_id
data << r.address_id
data << r.address_name
data << r.status
data << r.exp_dt
data << r.version
data << r.is_key_name
data << r.data_source
data << r.description
data << r.alias_grade
data << r.addr_type_cd
data << r.ifexamine
data << r.ifdisplay
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
}
def exportBigdataDealFJW(dealdate){
@@ -368,7 +419,7 @@ class ExportDistrictRongHe extends BaseBatch{
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def allRows = []
@@ -453,7 +504,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
}
@@ -468,31 +519,36 @@ class ExportDistrictRongHe extends BaseBatch{
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
if(curDate.toString().endsWith("28")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
"where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" +
" distid not in ${newstr} ";//CITYID='${cityCode}' and
def rets1 = createDeal(sql1)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
" where distid in ${newstr}"
def rets2 = createDeal(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
allRows = createDeal(sql)
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("28")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
"where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" +
" distid not in ${newstr} ";//CITYID='${cityCode}' and
def rets1 = createDeal(sql1)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
" where CITYID='${cityCode}' and distid in ${newstr}"
def rets2 = createDeal(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//
allRows = createDeal(sql)
}
}
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def createDeal(sql){
@@ -653,54 +709,68 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
println "BIGDATA_DEAL_FJW 导出完成......"
}
def exportBigdataBuildingFJW(cityCode){
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
def index = 0
for(def city : citys) {
def allRows = []
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
if(allRows && allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
@@ -724,40 +794,40 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_FJW 上传Ftp完成......"
println "EVALUATE_RATE_FJW 上传Ftp完成......"
}
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate)
@@ -70,7 +70,7 @@ class CommonDatautils {
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -136,6 +136,13 @@ class CommonDatautils {
return cityInfos;
}
public static def findAlCityDetail(city){
DBObject q = new BasicDBObject();
q.put("display", city)
return SERVER_AREA_COLL.findOne(q);
}
public static def findAllCityByGrade(def city_grade){
DBObject q = new BasicDBObject();
@@ -466,6 +473,7 @@ class CommonDatautils {
if(dbo.mapping_alias){
detailMap.mapping_alias = dbo.mapping_alias;
}
detailMap.estate_type = dbo.estate_type2
}
}catch(Exception e){
@@ -747,7 +755,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","森望芳草绿岸")
// q.put("name","湖畔佳苑")
DBObject keys = new BasicDBObject();
if(fields){
@@ -2042,6 +2050,20 @@ class CommonDatautils {
}
public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryDistrictPrices(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
@@ -3145,8 +3167,7 @@ class CommonDatautils {
DBObject dbo;
def prices = []
def index = 0
while(cur.hasNext()) {
def obj = cur.next()
for(def obj : cur) {
if(index == 0){
dbo = obj
}
@@ -3313,5 +3334,16 @@ class CommonDatautils {
def count = HOUSE_LOUDONG_COLL.count(q)
return count;
}
static def findCityInfo(city){
def q = new BasicDBObject()
q.put("city", city)
def cur = CITY_INFO_COLL.find(q).sort(new BasicDBObject("weekend",-1)).limit(1)
while(cur.hasNext()) {
DBObject bsobj = cur.next()
return bsobj;
}
return null
}
}
@@ -0,0 +1,159 @@
package com.fangjia.script.crawl
import com.alibaba.fastjson.JSONArray
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
class SchoolLocation {
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
static main(args) {
def task = new SchoolLocation()
task.run()
}
def run(){
def query = new BasicDBObject()
query.put("province","江苏")
//query.put("lat",null)
query.put("lat",new BasicDBObject('$exists',false))
def cursor = school_parse_coll.find(query)
def index = 0
for(def d : cursor){
index++
def datas = getSchoolLatAndLng(d.city, d.name, d.address, false)
if(datas){
for(def ret : datas){
def addr = d.address
if(d.address.contains(d.city+"市")){
def cityStr = d.city+"市"
addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size())
}
if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){
def udata = new BasicDBObject()
udata.put("lat", ret.lat.toString().toDouble())
udata.put("lng", ret.lng.toString().toDouble())
udata.put("status", 0)
school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
println index +" "+udata
break;
}
}
}
}
}
def getSchoolLatAndLng(city, name, address, isFullMatch){
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","").replace("?","").replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null;
com.alibaba.fastjson.JSONObject result = null;
try {
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
String response = request.get(url, params, "UTF-8");
com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response)
if (jsonResponse != null) {
JSONArray ja = jsonResponse.getJSONArray("pois");
if(ja != null && ja.size() > 0) {
poiList = new ArrayList<>();
for(int i=0;i<ja.size();i++){
com.alibaba.fastjson.JSONObject jo_item = ja.getJSONObject(i);
if(jo_item.containsKey("cityname") && !jo_item.getString("cityname").contains(city)){
continue;
}
String item_address = jo_item.containsKey("address") ? jo_item.getString("address") : null;
String item_name = jo_item.containsKey("name") ? jo_item.getString("name") : null;
boolean getedData = false;
if(isFullMatch == true){
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.equalsIgnoreCase(item_name, address)){
getedData = true;
}
}else{
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.contains(item_address, address)
|| StringUtils.equalsIgnoreCase(item_name, address) || StringUtils.contains(item_name, address) || StringUtils.contains(address, item_name)){
getedData = true;
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(item_name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(item_address, address);
if(score1 || score2){
getedData = true;
}
if(!getedData){
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(address, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(item_name, " ")
if(py1 == py2){
getedData = true;
}
}
}
}
def type = jo_item.getString("type");
if(getedData == true){
com.alibaba.fastjson.JSONObject jo = new com.alibaba.fastjson.JSONObject();
if(jo_item.containsKey("id")){
jo.put("id", jo_item.getString("id"))
}
if(jo_item.containsKey("name")){
jo.put("name", jo_item.getString("name"))
}
if(jo_item.containsKey("type")){
jo.put("type", type)
}
if(jo_item.containsKey("address")){
jo.put("address", jo_item.getString("address").replace("[]",""))
}
if(jo_item.containsKey("pname")){
jo.put("pname", jo_item.getString("pname"))
}
if(jo_item.containsKey("cityname")){
jo.put("cityname", jo_item.getString("cityname"))
}
if(jo_item.containsKey("adname")){
jo.put("adname", jo_item.getString("adname"))
}
if(jo_item.containsKey("location")){
def arrs = jo_item.getString("location").split(",")
jo.put("lat", arrs[1])
jo.put("lng", arrs[0])
}
if(type && type.contains("出入口")){
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】-${type}"
}else{
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】"
poiList.add(jo);
}
}else{
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
}
}
}else{
println "X 高德地图 ( "+city+"-"+address+" )无结果!"
}
}
} catch (Exception e) {
e.printStackTrace();
}
return poiList;
}
}
@@ -11,6 +11,10 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -26,7 +30,9 @@ class DistrictInfoExportBatch extends BaseBatch{
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new DistrictInfoExportBatch(), args)
@@ -87,7 +93,12 @@ class DistrictInfoExportBatch extends BaseBatch{
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
for(def rMap : districts){
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
saveOrUpdateFzDist(districts)
def fzDistricts = getFzDistricts(city)
for(def rMap : fzDistricts){
//¼Û¸ñûÓÐÒ²Ìṩ
// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
// if(dbo){
@@ -139,6 +150,51 @@ class DistrictInfoExportBatch extends BaseBatch{
return result
}
def saveOrUpdateFzDist(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dbo = FZ_DIST_COLL.findOne(new BasicDBObject("_id", d.get('_id')))
if(dbo){
def udate = new BasicDBObject()
d.each {k,v->
udate.put(k,v)
}
udate.removeField("_id")
udate.put("m_date", new Date())
FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate));
}else{
FZ_DIST_COLL.save(d)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
}
return rets
}
def processAlias(alias, def addr_alias){
def newAlias = []
if(alias){
@@ -29,7 +29,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def charset = "GBK"
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -86,10 +88,21 @@ class DistrictPriceExportBatch extends BaseBatch{
return sb.toString()
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
}
return rets
}
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
districts.addAll(removeDists)
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
// districts.addAll(removeDists)
def districts = getFzDistricts(city)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
@@ -124,9 +137,15 @@ class DistrictPriceExportBatch extends BaseBatch{
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
}else{
curPrice = specialDistsPriceMap.get(rMap.name)
if(curPrice){
def randMarkup = nextDouble(0, 1)
curPrice = (int)(curPrice*randMarkup/100 + curPrice)
}
}
def m = []
if(curPrice){
if(curPrice && dbo){
m << dbo.get("_id").toString()//¾ù¼ÛID
}else{
m << new ObjectId().toString()
@@ -135,7 +154,11 @@ class DistrictPriceExportBatch extends BaseBatch{
m << rMap.city
m << rMap.region
m << rMap.name
m << curPrice
if(curPrice){
m << curPrice
}else{
m << 0
}
m << curDate.toInteger();
result << m
}
@@ -24,7 +24,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.priceMarkupRun();
task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
@@ -88,14 +88,21 @@ class FzDataResultStat {
lastPrice = lastObj[5]
}
def markup = null
if(list && lastPrice){
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
try{
if(list && lastPrice && lastPrice.toInteger()>0){
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
}
}catch(Exception e){
e.printStackTrace()
}
def dist = CommonDatautils.findDistrictById(list[1])
def flag = "有效"
if(dist.status!=0){
if(dist && dist.status!=0){
flag = "失效"
}
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
@@ -131,7 +138,7 @@ class FzDataResultStat {
def regionPriceTrendRun(){
def city = "福州"
def add = -6;
def add = -9;
def months = []
while(add<0){
months << DateUtils.addMonth(new Date(), add)
@@ -0,0 +1,176 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.io.FileUtils
class FzDistrictMatch {
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new FzDistrictMatch()
// task.run();
// task.addPrice()
task.addThirdPrice()
}
def addThirdPrice(){
def map = read("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv");
ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d ->
if(d['G']){
def priceMap = map.get(d['G'])
def prices = []
if(priceMap && priceMap.zhugezhaofang && priceMap.zhugezhaofang.isNumber()){
d['J'] = priceMap.zhugezhaofang
prices << priceMap.zhugezhaofang.toInteger()
}
if(priceMap && priceMap.anjuke && priceMap.anjuke.isNumber()){
d['K'] = priceMap.anjuke
prices << priceMap.anjuke.toInteger()
}
if(priceMap && priceMap.soufang && priceMap.soufang.isNumber()) {
d['L'] = priceMap.soufang
prices << priceMap.soufang.toInteger()
}
if(priceMap && priceMap.lianjia && priceMap.lianjia.isNumber()){
d['M'] = priceMap.lianjia
prices << priceMap.lianjia.toInteger()
}
if(priceMap && priceMap.cityhouse && priceMap.cityhouse.isNumber()){
d['N'] = priceMap.cityhouse
prices << priceMap.cityhouse.toInteger()
}
if(priceMap && priceMap.fangtan && priceMap.fangtan.isNumber()){
d['O'] = priceMap.fangtan
prices << priceMap.fangtan.toInteger()
}
if(priceMap && priceMap.fangdoudou && priceMap.fangdoudou.isNumber()){
d['P'] = priceMap.fangdoudou
prices << priceMap.fangdoudou.toInteger()
}
if(priceMap && priceMap.maitian && priceMap.maitian.isNumber()){
d['Q'] = priceMap.maitian
prices << priceMap.maitian.toInteger()
}
if(priceMap && priceMap.get("58tc") && priceMap.get("58tc").isNumber()){
d['R'] = priceMap.get("58tc")
prices << priceMap.get("58tc").toInteger()
}
if(prices){
d['S'] = (int)(prices.sum()/prices.size())
}
}
},0)
}
def read(fileName){
List<String> list1= FileUtils.readLines(new File(fileName))
def map1 = [:]
for(def str : list1){
def s = str.split("@\\|@")
def key = s[2]+"##"+s[3]+"##"+s[4]
def priceMap = [:]
map1.put(s[1], priceMap)
if(s.size()>=7 && s[6]){
priceMap.put("zhugezhaofang", s[6])
}
if(s.size()>=8 && s[7]){
priceMap.put("anjuke", s[7])
}
if(s.size()>=9 && s[8]){
priceMap.put("soufang", s[8])
}
if(s.size()>=10 && s[9]){
priceMap.put("lianjia", s[9])
}
if(s.size()>=11 && s[10]){
priceMap.put("cityhouse", s[10])
}
if(s.size()>=12 && s[11]){
priceMap.put("fangtan", s[11])
}
if(s.size()>=13 && s[12]){
priceMap.put("fangdoudou", s[12])
}
if(s.size()>=14 && s[13]){
priceMap.put("maitian", s[13])
}
try{
if(s.size()>=15 && s[14]){
priceMap.put("58tc", s[14])
}
}catch(Exception e){
e.getMessage()
}
}
return map1
}
def addPrice(){
ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d ->
if(d['G']){
def avg_price = findFzDistrictStat( d['G'],202207)
d['F'] = avg_price
}
},0);
}
def findFzDistrictStat(id,lastMonthDate){
def q = new BasicDBObject()
q.put("comm_id", id)
q.put("month", lastMonthDate)
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
for(def d : dbcursor){
return d.avg_price
}
return null
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx",{d->
def region = d['C']
if(!d['G'] && region){
def name = d['B'].toString().replace("(","").replace(")","")
// name = d['B'].toString().replace("(非钢混)","").replace("(钢混)","")
region = RegionData.formatRegion("福州", region)
// def dist = CommonDatautils.findDistrict("福州", region, name, "district")
// if(dist){
// d['G'] = dist._id.toString()
// d['H'] = dist.name
// }
def jsonObject = OpenApiClient.addressMatch("福州", name, "property", false, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
d['H'] = jsonObject.get("name")
d['G'] = jsonObject.get("id")
d['J'] = jsonObject.get("district")
}
}
},0)
}
}
@@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{
// int endDate = 20191201
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd")
// def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
def endDate = "20220710"
def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
// QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false);
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202209.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -75,81 +75,98 @@ class ThirdPriceExportBatch extends BaseBatch{
row << dist.region
row << dist.name
row << o.s_date
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
row << zgf_price
if(zgf_price){
add = true;
}
// if(o.zhugezhaofang_esf_price){
// add = true;
// }
def ajk_price = getLastThirdPrice(o.anjuke_esf_price, curPrice)
row << ajk_price
if(ajk_price){
add = true;
}
// if(o.anjuke_esf_price){
// add = true;
// }
def sf_price = getLastThirdPrice(o.soufang_esf_price, curPrice)
row << sf_price
if(sf_price){
add = true;
}
// if(o.soufang_esf_price){
// add = true;
// }
def lj_price = getLastThirdPrice(o.lianjia_esf_price, curPrice)
row << lj_price
if(lj_price){
add = true;
}
// if(o.lianjia_esf_price){
// add = true;
// }
def city_price = getLastThirdPrice(o.cityhouse_esf_price, curPrice)
row << city_price
if(city_price){
add = true;
}
// if(o.cityhouse_esf_price){
// add = true;
// }
def ft_price = getLastThirdPrice(o.fangtan_esf_price, curPrice)
row << ft_price
if(ft_price){
add = true;
}
// if(o.fangtan_esf_price){
// add = true;
// }
def fdd_price = getLastThirdPrice(o.fangdoudou_esf_price, curPrice)
row <<fdd_price
if(fdd_price){
add = true;
}
// if(o.fangdoudou_esf_price){
// add = true;
// }
def mt_price = getLastThirdPrice(o.maitian_esf_price, curPrice)
row << mt_price
if(mt_price){
add = true;
}
// if(o.maitian_esf_price){
// add = true;
// }
def tc_price = getLastThirdPrice(o.get("58tc_esf_price"), curPrice)
row << tc_price
if(tc_price){
add = true;
}
// row << o.zhugezhaofang_esf_price
// if(o.zhugezhaofang_esf_price){
// add = true;
// }
// row << o.anjuke_esf_price
// if(o.anjuke_esf_price){
// add = true;
// }
// row << o.soufang_esf_price
// if(o.soufang_esf_price){
// add = true;
// }
//
// row << o.lianjia_esf_price
// if(o.lianjia_esf_price){
// add = true;
// }
//
// row << o.cityhouse_esf_price
// if(o.cityhouse_esf_price){
// add = true;
// }
//
// row << o.fangtan_esf_price
// if(o.fangtan_esf_price){
// add = true;
// }
// row << o.fangdoudou_esf_price
// if(o.fangdoudou_esf_price){
// add = true;
// }
// row << o.maitian_esf_price
// if(o.maitian_esf_price){
// add = true;
// }
//
// row << o.get("58tc_esf_price")
// if(o.get("58tc_esf_price")){
// add = true;
// }
@@ -5,6 +5,7 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -25,14 +26,15 @@ class NoHouseDistrictStatBatch extends BaseBatch{
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static EsDataSource dataSource = new EsDataSource()
// public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static EsDataSource dataSource = null
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
dataSource = ElasticUtils.getEsDataSource("seawater.es.db.fangjia.com")
// dataSource.setUrl(address)
// Properties connectionProperties = new Properties()
// connectionProperties.put("user", "developer")
// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
// dataSource.setProperties(connectionProperties)
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -18,9 +18,10 @@ class RegionUpdate {
}
def run1(){
def districts = CommonDatautils.findAllFiledDistrict("district","ÕÄÖÝ");
def city = "ËÄÆ½"
def districts = CommonDatautils.findAllFiledDistrict("district",city);
def regions = CommonDatautils.getRegionListByCity("ÕÄÖÝ")
def regions = CommonDatautils.getRegionListByCity(city)
def index = 0
def addIndex = 0
DistrictRegionTransfer transfer = new DistrictRegionTransfer()
@@ -28,6 +29,9 @@ class RegionUpdate {
if(districts){
//根据经纬度获取经纬度所在区域
for(def dist : districts){
if(dist.region!="¹«Ö÷Áë"){
continue
}
index++
if(!regions.contains(dist.region)){
@@ -35,28 +39,28 @@ class RegionUpdate {
println regionErrorIndex +" "+dist.name
}
// def region = getRegionByLocation(dist.lng/10000000000d+","+dist.lat/10000000000d);
// if(!region){
// def detailMap = getRegionByAddress("ÕÄÖÝ", dist.address);
// if(detailMap && detailMap.region && detailMap.region != dist.region){
// addIndex++
// dist.lng = (detailMap.lng.toDouble()*10000000000).longValue()
// dist.lat = (detailMap.lat.toDouble()*10000000000).longValue()
//
// def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble())
// dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
// dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
//
// println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name
// transfer.transfer(dist, dist.city, detailMap.region)
// }
// }else{
// if(region && region!=dist.region){
// addIndex++
// println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name
// transfer.transfer(dist, dist.city, region)
// }
// }
def region = getRegionByLocation(city, dist.lng/10000000000d+","+dist.lat/10000000000d);
if(!region){
def detailMap = getRegionByAddress(city, dist.address);
if(detailMap && detailMap.region && detailMap.region != dist.region){
addIndex++
dist.lng = (detailMap.lng.toDouble()*10000000000).longValue()
dist.lat = (detailMap.lat.toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble())
dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name
transfer.transfer(dist, dist.city, detailMap.region)
}
}else{
if(region && region!=dist.region){
addIndex++
println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name
transfer.transfer(dist, dist.city, region)
}
}
}
}
}
@@ -85,14 +89,16 @@ class RegionUpdate {
return detail
}
def getRegionByLocation(def location){
def getRegionByLocation(city, def location){
def json = GaoDeApiUtils.geocode(location);
def city = "ÕÄÖÝ"
def region = null;
if (json && json.info=="OK") {
def regeocode = json.regeocode
if(regeocode && regeocode.addressComponent){
region = regeocode.addressComponent.district
if(!region){
region = regeocode.addressComponent.township
}
}
}
if(region){
@@ -48,13 +48,13 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-06-01"
static def statDate = "2022-09-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
}
@@ -284,8 +284,8 @@ class YouDaDataParser {
}
def esfParser(){
def files = ["D:\\数据\\友达\\2022年7月\\查询1.DBF"]
def esfParser(path){
def files = [path]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -1,4 +1,4 @@
package com.fangjia.script.gj.test
package com.fangjia.script.gj.commStat
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -28,11 +28,30 @@ class CityStat {
def task = new CityStat()
// task.run()
// task.run1();
task.run3();
// task.run3();
// task.run4()
// task.run6()
// task.run7()
// task.run8()
task.dealPriceStat()
}
def dealPriceStat(){
//def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
def citys = ["石家庄","贵阳"]
def rets = []
rets << ["城市","2022-4月总量"]
def startDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
def endDate = DateUtils.parseFromTightByPattern('2022-02-01','yyyy-MM-dd')
for(def c : citys){
def ret = []
ret << c
ret << findDealPriceCount(c, startDate, endDate)
rets << ret
println ret
}
ExcelUtils.write("D:\\数据\\2021成交数据.xlsx", rets, "月统计")
}
// def run8(){
@@ -464,6 +483,14 @@ class CityStat {
return count;
}
def findDealPriceCount(def city, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
def count = DEAL_PRICE_COLL.count(q)
return count;
}
def calTradeDistCount(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
@@ -1,4 +1,4 @@
package com.fangjia.script.gj.test
package com.fangjia.script.gj.commStat
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
@@ -29,24 +29,24 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
for(def city : citys){
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//for(def city : citys){
//
//}
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\江苏数据统计.xlsx",{d->
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
// if(keys){
// d['E'] = keys.size()
//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
//// if(keys){
//// d['G'] = keys.size()
//// }
//
//// def rets = findHousesByCity(d['A'], "sell", 20210701)
// if(d['A']){
// def count = findHouseCount(d['A'])
// d['H'] = count
// }
// def rets = findHousesByCity(d['A'], "sell", 20210701)
if(d['A']){
def count = findHouseCount(d['A'])
d['F'] = count
}
})
//
//})
def findHouseCount(city){
@@ -288,7 +288,7 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
//}
//
//def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
//def rows = []
//rows << ["城市","小区量", "别名量", "地址别名量"]
//for(def c : citys){
@@ -305,7 +305,24 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
// }
// rows << [c, districts.size(), alias_count, addr_alias_count]
//}
//
//
//
//ExcelUtils.write("D:\\数据\\江苏\\江苏数据统计.xlsx", rows, "结果")
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
//区域级统计
def city = "合肥"
def regions = CommonDatautils.getCityRegions(city)
def rows = []
for(def region : regions){
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
def alias_count = 0
def addr_alias_count = 0
for(def r : districts){
if(r.alias && r.alias.size()>0){
alias_count ++
}
if(r.addr_alias && r.addr_alias.size()>0){
addr_alias_count++
}
}
rows << [region, districts.size(), alias_count, addr_alias_count]
}
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
@@ -110,6 +110,7 @@ class FangjiaClientUtils {
public static def saveToEsPaimai(def data){
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr)
}
@@ -0,0 +1,158 @@
package com.fangjia.script.gj.dongguanbank
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import org.apache.lucene.queryparser.classic.QueryParser
class DongGuanBankTest {
static main(args) {
def task = new DongGuanBankTest()
def path = "D:\\数据\\东莞银行\\东莞银行测试_输出单价(2).xlsx"
// task.run(path)
// task.addPrice(path)
task.addMinMaxPrice(path)
}
def run(path){
ExcelUtils.multiThreadingModify(path,{d->
def city = d['A'].replace("市","")
if(city){
def region = RegionData.formatRegion(city, d['B'])
def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C'])
if(results && results.precise>=0){
d['H'] = results.city
d['I'] = results.district
d['J'] = results.name
d['Q'] = results.propertytype
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d ->
if(d['H']){
def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district")
def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
if(priceObj){
d['M'] = priceObj.avg_price
d['N'] = (priceObj.avg_price * d['G']).toInteger()
d['O'] = ((int)(priceObj.avg_price/100))*100
if(dist){
d['R'] = dist.estate_type2
}
if(priceObj.value_type){
if(priceObj.value_type == 1){
d['S'] = "指导价"
}else if(priceObj.value_type == 2){
d['S'] = "自身样本"
}else if(priceObj.value_type == 3){
d['S'] = "第三方"
}else if(priceObj.value_type == 4){
d['S'] = "交易数据"
}else if(priceObj.value_type == 5){
d['S'] = "周边楼盘"
}else if(priceObj.value_type == 6){
d['S'] = "推导"
}else if(priceObj.value_type == 9){
d['S'] = "神经网络"
}else if(priceObj.value_type == 10){
d['S'] = "无方式"
}
}
}
if(!d['P']){
def alias = []
if(!dist.alias){
alias = []
}else{
alias = dist.alias
}
alias << dist.name
def map = findMaxAvgPrice(dist.city, dist.region, alias)
if(map){
d['P'] = map.max
d['Q'] = map.min
}
}
}
})
}
def findMaxAvgPrice(def city, def region, def alias){
def set = null
try{
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210901 and region='${region}' and district_name in ('${aliasStr}') and root='anjuke.com'"
set = new HashSet()
ElasticUtils.executeSqlQuery(sql,{data->
set << data.avg_price
})
}catch(Exception e){
e.printStackTrace()
}
if(set && set.size()>=3){
def map = [:]
set = set.sort()
map.min = set[0]
map.max = set[set.size()-1]
return map
}
return null
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findMinPrice(city){
def avg_price = null
ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", {
avg_price = it.avg_price
})
return avg_price
}
def addMinMaxPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['V']){
def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district")
if(dist){
def alias = []
if(!dist.alias){
alias = []
}else{
alias = dist.alias
}
alias << dist.name
def map = findMaxAvgPrice(dist.city, dist.region, alias)
if(map){
d['V'] = map.max
d['W'] = map.min
}
}
}
})
}
}
@@ -19,9 +19,44 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
task.houseRun()
// task.houseRun()
// task.statHouseUpdateCount()
task.statDistHouse()
}
def statDistHouse(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\大连有效小区确认_2256--查好-1.xlsx",{d->
if(d['B']){
def alias = []
if(d['G']){
def s = d['G'].replace("[", "").replace("]", "").replace("'","").replace('"',"")
def arrs = null
if(s){
arrs = s.split(",")
}
if(arrs){
arrs = arrs as List
if(!arrs.contains(d['D'])){
arrs << d['D']
}
alias = arrs
}else{
alias << d['D']
}
}else{
alias << d['D']
}
def distCount = findHouseCount(d['B'], d['C'], alias, 20190101, 20221001)
if(distCount){
d['E'] = distCount
}
println d['B']+" "+d['C']+" "+d['D']+" "+distCount
}
})
}
def run(){
@@ -0,0 +1,55 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
findLogs()
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
def q = QueryBuilders.fuzzyQuery("request", "/jnBank")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
if(houses) {
for (def d : houses) {
def request = d.request
if(request){
def arrs = request.split("&")
if(arrs){
for(def a : arrs){
if(a.contains("CITY_CD")){
def list = a.split("=")
if(list && list.size()>=2){
set.add(list[1])
}
}
}
}
}
}
}
println set
// ElasticUtils.executeSqlQuery(sql,{ data->
// println data.request
// },"192.168.0.25")
}catch(Exception e){
e.printStackTrace()
}
return total
}
@@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
// calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
@@ -223,6 +223,7 @@ class DistrictIndexBatch extends BaseBatch{
map.city = d.city
map.region = d.region
map.name = d.name
map.status = 0
map.avg_estate_charge = rMap.avg_estate_charge
map.price_rank = rMap.price_rank
@@ -44,12 +44,20 @@ public class BeiKeCrawlByChrome {
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
citys = citys.reverse()
// citys = citys.reverse()
// def index = 0
// for(def c : citys){
// index++
// if(c.city=="合肥"){
// break
// }
// }
// citys = citys[index..citys.size()-1]
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
@@ -104,9 +112,23 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, m.city, m.url)
println m.city +" ÏêÇéÁ´½Ó£º"+detailUrls.size()
driver.get(m.url+"/chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = m.url+ r.attr("href")
println "开始抓取【${m.city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, m.city, href, regionText)
println m.url +" 详情链接:"+detailUrls.size()
}
}
// def detailUrls = getPageDeail(driver, m.city, m.url)
// println m.city +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -142,12 +164,12 @@ public class BeiKeCrawlByChrome {
}
def getPageDeail(driver, city, cityUrl){
def getPageDeail(driver, city, cityUrl, region){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "/chengjiao/pg${page}/"
def url = cityUrl + "pg${page}/"
try{
driver.get(url)
}catch(Exception e){
@@ -164,7 +186,7 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
parse(city, e, region)
}else{
stop = true;
break;
@@ -176,13 +198,13 @@ public class BeiKeCrawlByChrome {
}
page++;
}else{
stop = true;
stop = true;
}
}
return detailUrls
}
def parse(city,e){
def parse(city,e, region){
try {
def title = e.select(".title a").text()
if (title) {
@@ -190,7 +212,7 @@ public class BeiKeCrawlByChrome {
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
@@ -376,13 +398,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("¸£ÖÝ")){
// if(e.text().contains("合肥")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
}
// cityMap.put(e.text(), "https:"+href)
// }
}
}
return list
@@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.NumberUtils
import java.util.Map.Entry;
import java.util.regex.Pattern
@@ -175,22 +177,23 @@ class ExportPaimai {
def run(){
def city = "宿迁"
def city = "合肥"
def region = null
def datas = findPaimaiData(city,region)
println "总量:"+datas.size()
def rows = []
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型",
"映射资产类型","映射拍卖状态","URL"]
rows << header
// def set = new HashSet()
def set = new HashSet()
def index = 0
for(def d : datas){
// if(!set.contains(d.title)){
if(!set.contains(d.title)){
index++
println index
// set << d.title
set << d.title
def row = []
row << d.title.replace(",",",")
row << d.province
@@ -226,14 +229,19 @@ class ExportPaimai {
row << d.lookCount
row << d.callCount
row << d.delayCount
row << d.url
row << d.rateDiscount//折价率
row << d.mappingEstateType;//映射类型
row << d.zcType;//拍卖类型:资产拍卖 司法拍卖
row << d.mappingAgencyName//映射资产类型
row << d.mappingStatus//映射拍卖状态
row << d.url
rows << row
// }else{
// println d.title
// }
}else{
println d.title +" "+d.url
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}).xlsx",rows,"数据")
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据")
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
println "生成完成........"
@@ -452,18 +460,41 @@ class ExportPaimai {
if(region){
pmquery.put("region",region)
}
// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
pmquery.put("url", new BasicDBObject('$regex', pattern))
// pmquery.put("url", new BasicDBObject('$regex', pattern))
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
def rets = []
for(def db : dbcursor){
rets << db
}
// Collections.sort(rets, new Comparator<BasicDBObject>() {
// @Override
// int compare(BasicDBObject o1, BasicDBObject o2) {
// if(o1.finishDate && o2.finishDate){
// return -o1.finishDate.compareTo(o2.finishDate)
// }else{
// return 0
// }
//
// }
// })
rets.sort {a,b->
if(a.finishDate && b.finishDate){
def v1 = a.finishDate
def v2 = b.finishDate
return v1.compareTo(v2)
}else{
return 0
}
}
rets = rets.reverse()
return rets
}
}
@@ -31,12 +31,13 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
task.run()
// task.run1()
task.addDealRegion();
// task.addDealRegion();
task.standardRegion()
// task.updadteDealPriceField()
}
@@ -116,7 +117,7 @@ public class LianjiaCrawlByChrome {
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
@@ -129,7 +130,7 @@ public class LianjiaCrawlByChrome {
def standardRegion(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
@@ -152,8 +153,9 @@ public class LianjiaCrawlByChrome {
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
// q.put("city", "福州")
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
@@ -187,8 +189,19 @@ public class LianjiaCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
driver.get(cityUrl+"chengjiao/")
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -224,12 +237,12 @@ public class LianjiaCrawlByChrome {
}
def getPageDeail(driver, city, cityUrl){
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
def url = cityUrl + "pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
@@ -238,10 +251,10 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
parse(city, e, regionText)
}else{
stop = true;
break;
@@ -259,7 +272,7 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e){
def parse(city,e, regionText){
try {
def title = e.select(".title a").text()
if (title) {
@@ -267,7 +280,11 @@ public class LianjiaCrawlByChrome {
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
@@ -275,50 +292,61 @@ public class LianjiaCrawlByChrome {
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -446,7 +474,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
@@ -8,8 +8,10 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
@@ -29,18 +31,20 @@ class PaimaiDataSend {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new PaimaiDataSend()
// task.jdRun()
// task.jdRunByMongo()
// task.aliZcRun()
// task.aliZcRun2()
task.aliZcRun2()
// task.aliFpRun()
// task.rmfysszcRun()
// task.jdRun()
// task.checkUrl()
task.paimaiSendToEs()
// task.paimaiSendToEs()
}
@@ -64,7 +68,9 @@ class PaimaiDataSend {
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
q.put("province","江苏省")
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -95,11 +101,22 @@ class PaimaiDataSend {
}
}
def jdRun(){
def jdRunByMongo(){
def pm_q = new BasicDBObject()
pm_q.put("city","宿迁")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
pm_q.put("mappingStage", null)
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
sendJd(d.url)
}
}
def sendJd(url){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁市")
// q.put("url","https://paimai.jd.com/108245329")
q.put("url", url)
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -110,6 +127,7 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
@@ -131,20 +149,97 @@ class PaimaiDataSend {
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
def jdRun(){
def index = 0
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
q.put("province","江苏")
// q.put("parser_status", new BasicDBObject('$ne',3))
// q.put("url","https://paimai.jd.com/115669796")
def dbcursor = jd_coll.find(q).batchSize(3000)
def rets = []
for(def d : dbcursor){
index++
println "jd:"+index
JSON json = new JSONObject()
json.put("project", "paimai_jd")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("content_json", d.content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content.content)
json.put("result", result)
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
sendJD(rets)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
sendJD(rets)
}
}
def sendJD(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city","苏州")
pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("status", null)
// pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
pm_q.put("province","江苏")
// pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
pm_q.put("mDate", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2022-10-01")))
Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("mappingEstateType", null)
// pm_q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
def flag = sendAliZc(d.url)
// def flag = sendAliZc(d.url)
def flag = sendAli(d.url)
if(!flag){
println "²»´æÔÚ:"+d.url
}
@@ -185,7 +280,6 @@ class PaimaiDataSend {
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
@@ -194,19 +288,15 @@ class PaimaiDataSend {
return flag
}
def aliFpRun(){
def sendAli(def url){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁")
q.put("url","https://sf-item.taobao.com/sf_item/631616791778.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
def rets = []
q.put("url",url)
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def flag = false
for(def d : dbcursor){
index++
flag = true
println "ali-fp: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
@@ -227,11 +317,76 @@ class PaimaiDataSend {
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
rets << json
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
println "共:"+rets.size()
return flag
}
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","江苏")
q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',2))
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
// q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = ali_fp_coll.find(q).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
index++
println "ali-fp: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
try{
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
}catch(Exception e){
println "trades结果格式问题 "+d.url
}
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
rets << json
if(rets.size()==5000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
}
def send(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
@@ -244,7 +399,7 @@ class PaimaiDataSend {
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("parser_status", 0)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
@@ -263,12 +418,21 @@ class PaimaiDataSend {
def aliZcRun2(){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁")
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
// q.put("city", "上海")
q.put("province","江苏")
q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","¹ÃËÕ")
// q.put("url","https://zc-item.taobao.com/auction/670178084823.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
index++
if(!d.url){
println d._id.toString()
continue
}
println "ali-zc: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao_zc")
@@ -276,6 +440,7 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.content)
result.put("sub_type", d.sub_type)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("sub_count", d.sub_count)
@@ -290,19 +455,52 @@ class PaimaiDataSend {
result.put("region", d.region)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 4)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
rets << json
if(rets.size()==1000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
// println "共:"+rets.size()
//
// def totalCount = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(rets.size())
// for(def i=0;i<rets.size();i++){
// final def json = rets[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try{
// totalCount.addAndGet(1)
// byte[] packedMsg = MessageUtils.pack(json)
// CommonUtils.sendMessage("result2parse2dev", packedMsg)
//
//
// def udata = new BasicDBObject()
// udata.put("parser_status", 1)
// udata.put("m_date", new Date())
// ali_zc_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
// println "send: "+totalCount.get()
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// });
// }
// latch.await()
}
def paimaiSendToEs(){
def rets = getPaimaiDatas()
println "共:"+rets.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
@@ -312,7 +510,7 @@ class PaimaiDataSend {
try{
totalCount.addAndGet(1)
def flag = FangjiaClientUtils.saveToEsPaimai(m)
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
println totalCount.get()+" "+m.city +" "+m.title +" "+flag
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -326,56 +524,81 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","江苏")
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-10")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
def o = [:]
o.addPrice = d.addPrice
o.agencyName = d.agencyName
o.area = d.area
o.buildYear = d.buildYear
o.callCount = d.callCount
o.city = d.city
o.court = d.court
o.delayCount = d.delayCount
o.delayWeekly = d.delayWeekly
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evaluatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
o.id = d.id
o.introduce = d.introduce
if(d.city){
def o = [:]
o.addPrice = d.addPrice
o.agencyName = d.agencyName
o.area = d.area
o.buildYear = d.buildYear
o.callCount = d.callCount
o.city = d.city
o.court = d.court
o.delayCount = d.delayCount
o.delayWeekly = d.delayWeekly
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evalatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
o.id = d.id
o.introduce = d.introduce
// o.location = d.location
o.lookCount = d.lookCount
o.marketPrice = d.marketPrice
o.matchAddress = d.matchAddress
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
o.province = d.province
o.rateDiscount = d.rateDiscount
o.region = d.region
o.roomNum = d.roomNum
o.sellOffAdvancePay = d.sellOffAdvancePay
o.source = d.source
o.stage = d.stage
o.startDate = d.startDate
o.startPrice = d.startPrice
o.status = d.status
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
rets << o
o.lookCount = d.lookCount
o.marketPrice = d.marketPrice
o.matchAddress = d.matchAddress
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.matchSeaweedId = d.matchSeaweedId
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
o.province = d.province
o.rateDiscount = d.rateDiscount
o.region = d.region
o.roomNum = d.roomNum
o.sellOffAdvancePay = d.sellOffAdvancePay
o.source = d.source
o.stage = d.stage
o.startDate = d.startDate
o.startPrice = d.startPrice
o.status = d.status
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.tradeAvgPrice = d.tradeAvgPrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
if(d.url.contains("taobao.com")){
o.source = "淘宝"
}else if(d.url.contains("jd.com")){
o.source = "京东"
}else if(d.url.contains("rmfysszc.gov.cn")){
o.source = "人民法院讼诉网"
}
o.rateDiscount = d.rateDiscount//折价率
o.mappingEstateType = d.mappingEstateType;//映射类型
o.paiMaiType = d.paiMaiType;//拍卖类型:资产拍卖 司法拍卖
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
o.mappingStatus = d.mappingStatus//映射拍卖状态
o.mappingStage = d.mappingStage
if(d.startRateDiscount){
o.startRateDiscount = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, d.startRateDiscount);//起拍折价率 起拍折价率 = 起拍价 / 市场价或评估价
}
o.diffPrice = d.diffPrice;//差价 差价 = 成交价-市场价或评估价
rets << o
}
}
return rets;
}
@@ -1,17 +1,79 @@
package com.fangjia.script.gj.region
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class MapAreaUpdate {
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new MapAreaUpdate()
task.run()
//task.run()
task.checkCcbRegion()
}
def checkCcbRegion(){
def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\建行最终区域编码.xlsx",0,1)
def citys = CcbCommonUtils.citys
def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
def fjw_citys = []
for(def d : datas){
if(!fjw_citys.contains(d['A'])){
fjw_citys << d['A']
}
}
// for(def fjwCity : fjw_citys){
// if(!citys_252.contains(fjwCity) && !citys.contains(fjwCity)){
// println fjwCity
// }
// }
for(def newCity : citys_252){
if(!fjw_citys.contains(newCity)){
println fjwCity
}
}
// //是否全在之前92个城市中
// def ccb_92_citys = getCcb92City(sqlInstance)
// for(def m : ccb_92_citys){
// def isExist = false
// for(def d : datas){
// if(m.cityId==d['E'].toInteger().toString() && m.distId==d['F'].toInteger().toString()){
// isExist = true;
// break
// }
// }
// if(!isExist){
// println m.cityId +" "+ m.distId +" 不存在"
// }
// }
}
def getCcb92City(sqlInstance){
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj"
def list = []
sqlInstance.eachRow(sql, { r ->
def m = [:]
m.region = r.DISTNAME
m.cityId = r.CITYID
m.distId = r.DISTID
list << m
})
return list;
}
def run(){
def rets = findAllCity()
for(def city : rets){
@@ -0,0 +1,214 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
class MingShengStat {
def cityLevel0 = ["北京","上海","广州","深圳"]
def cityLevel1 = ["成都","重庆","杭州","武汉","西安","郑州","青岛","长沙","天津","苏州","南京","东莞","沈阳","合肥","佛山"]
def cityLevel2 = ["昆明","福州","无锡","厦门","哈尔滨","长春","南昌","济南","宁波","大连","贵阳","温州","石家庄","泉州","南宁","金华","常州","珠海","惠州","嘉兴","南通","中山","保定","兰州","台州","徐州","太原","绍兴","烟台","廊坊"]
def cityLevel3 = ["海口","汕头","潍坊","扬州","洛阳","乌鲁木齐","临沂","唐山","镇江","盐城","湖州","赣州","漳州","揭阳","江门","桂林","邯郸","泰州","济宁","呼和浩特","咸阳","芜湖","三亚","阜阳","淮安遵义","银川","衡阳","上饶","柳州","淄博","莆田","绵阳","湛江","商丘","宜昌",
"沧州","连云港","南阳","蚌埠","驻马店","滁州","邢台","潮州","秦皇岛","肇庆","荆州","周口","马鞍山","清远","宿州","威海","九江","新乡","信阳","襄阳","岳阳","安庆","菏泽","宜春","黄冈","泰安","宿迁","株洲","宁德","鞍山","南充","六安","大庆","舟山"]
def cityLevel4 = ["常德","渭南湖","孝感","丽水","运城","德州","张家口","鄂尔多斯","阳江","泸州","丹东","曲靖","乐山","许昌","湘潭","晋中","安阳","齐齐哈尔","北海","宝鸡","抚州","景德镇","延安","三明","抚顺亳州","日照","西宁","衢州","拉萨","淮北","焦作","平顶山","滨州","吉安","濮阳","眉山","池州","荆门","铜仁","长治",
"衡水","铜陵","承德","达州","邵阳","德阳","龙岩","南平","淮南","黄石","营口","东营","吉林","韶关","枣庄","包头","怀化","宣城","临汾","聊城","梅州","盘锦","锦州","榆林","玉林","十堰","汕尾","咸宁","宜宾","永州","益阳","黔南州","黔东南","恩施","红河","大理","大同","鄂州","忻州","吕梁","黄山","开封","郴州","茂名","漯河","葫芦岛","河源","娄底","延边"]
def cityLevel5 = ["汉中","辽阳","四平","内江","六盘水","安顺","新余","牡丹江","晋城","自贡","三门峡","赤峰","本溪","防城港","铁岭","随州","广安","广元","天水","遂宁","萍乡","西双版纳","绥化","鹤壁","湘西","松原","阜新","酒泉","张家界","黔西南","保山","昭通","河池","来宾","玉溪","梧州","鹰潭","钦州","云浮","佳木斯","克拉玛依","呼伦贝尔","贺州",
"通化","阳泉","朝阳","百色","毕节","贵港","丽江","安康","通辽","德宏","朔州","伊犁","文山","楚雄","嘉峪关","凉山","资阳","锡林郭勒盟","雅安","普洱","崇左","庆阳","巴音郭楞(巴州)","乌兰察布","白山","昌吉","白城","兴安盟","定西","喀什","白银","陇南","巴彦淖尔","巴中","鸡西","乌海","临沧","海东","张掖","商洛","黑河","哈密","吴忠","阿克苏"]
def ajkNewDistsMap = ["北京": 538, "上海": 1928, "广州": 1834, "深圳": 1149, "武汉": 2181, "大连": 866, "南京": 1442, "杭州": 2245, "太原": 497, "石家庄": 723, "重庆": 2700, "西安": 1693, "福州": 305, "济南": 1003, "宁波": 1304, "成都": 3004, "天津": 1614, "昆明": 755, "南昌": 697, "青岛": 1843, "厦门": 253, "郑州": 1385, "长沙": 1811, "长春": 741, "合肥": 1265, "呼和浩特": 462, "沈阳": 456, "贵阳": 499, "拉萨": 82, "宝鸡": 249, "汕头": 411, "苏州": 1531, "温州": 566, "泉州": 565, "邯郸": 314, "唐山": 442, "沧州": 248, "洛阳": 560, "南阳": 422, "襄阳": 302, "吕梁": 79, "大同": 171, "潍坊": 739, "南通": 735, "衡阳": 425, "中山": 590, "宜昌": 360, "许昌": 384, "盐城": 459, "济宁": 510, "龙岩": 227, "上饶": 288, "马鞍山": 199, "赣州": 385, "金华": 479, "常德": 394, "延边": 71, "泰州": 304, "三亚": 456, "淮安": 488, "咸宁": 307, "无锡": 703, "常州": 470, "镇江": 382, "烟台": 917, "绍兴": 714, "莆田": 261, "东莞": 729, "佛山": 1161, "株洲": 285, "黄石": 151, "孝感": 379]
def bkNewDistsMap = ["北京": 316, "上海": 2002, "广州": 1350, "深圳": 1261, "武汉": 2308, "大连": 438, "南京": 579, "杭州": 1892, "太原": 418, "石家庄": 582, "重庆": 2122, "西安": 1608, "福州": 512, "济南": 779, "宁波": 592, "成都": 2849, "天津": 1039, "昆明": 1262, "南昌": 429, "青岛": 1275, "厦门": 387, "郑州": 1146, "长沙": 1434, "长春": 433, "合肥": 807, "呼和浩特": 294, "沈阳": 1137, "贵阳": 242, "拉萨": 3, "宝鸡": 50, "苏州": 1129, "温州": 126, "泉州": 410, "邯郸": 114, "唐山": 181, "沧州": 50, "洛阳": 192, "南阳": 290, "襄阳": 55, "大同": 62, "潍坊": 332, "南通": 509, "衡阳": 83, "中山": 516, "宜昌": 147, "许昌": 156, "盐城": 211, "济宁": 205, "龙岩": 5, "上饶": 160, "马鞍山": 120, "赣州": 236, "金华": 124, "常德": 116, "三亚": 164, "淮安": 278, "咸宁": 75, "无锡": 575, "常州": 221, "镇江": 310, "烟台": 548, "绍兴": 175, "莆田": 13, "东莞": 622, "佛山": 971, "株洲": 134, "黄石": 88, "孝感": 94]
def ljNewDistMap = ["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40]
def sfNewDistMap =["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40]
def chNewDistMap = ["北京": 408, "上海": 532, "广州": 559, "深圳": 763, "武汉": 775, "大连": 7, "南京": 1137, "杭州": 517, "太原": 293, "石家庄": 567, "重庆": 817, "西安": 439, "济南": 544, "宁波": 555, "成都": 1094, "天津": 754, "昆明": 446, "南昌": 379, "青岛": 840, "厦门": 267, "郑州": 960, "长沙": 767, "长春": 299, "合肥": 404, "呼和浩特": 136, "沈阳": 385, "贵阳": 330, "拉萨": 41, "宝鸡": 145, "汕头": 220, "苏州": 715, "温州": 422, "泉州": 312, "邯郸": 425, "唐山": 317, "沧州": 298, "洛阳": 268, "南阳": 233, "襄阳": 84, "吕梁": 38, "大同": 121, "潍坊": 421, "南通": 328, "衡阳": 329, "中山": 279, "宜昌": 158, "许昌": 161, "盐城": 165, "济宁": 251, "龙岩": 90, "上饶": 136, "马鞍山": 69, "赣州": 265, "金华": 252, "常德": 192, "延边": 59, "泰州": 122, "三亚": 120, "淮安": 150, "咸宁": 175, "无锡": 468, "常州": 259, "镇江": 223, "烟台": 420, "绍兴": 368, "莆田": 106, "东莞": 249, "佛山": 438, "株洲": 216, "黄石": 96, "孝感": 160]
def jwNewDistMap = ["北京": 510, "上海": 1353, "广州": 1193, "深圳": 697, "武汉": 1746, "大连": 539, "南京": 1219, "杭州": 2030, "太原": 631, "石家庄": 521, "重庆": 2165, "西安": 1518, "济南": 811, "宁波": 1023, "成都": 3192, "天津": 91, "昆明": 764, "南昌": 632, "青岛": 1420, "厦门": 274, "郑州": 1509, "长沙": 1474, "长春": 400, "合肥": 1112, "呼和浩特": 320, "沈阳": 648, "贵阳": 501, "拉萨": 75, "宝鸡": 217, "汕头": 350, "苏州": 1210, "温州": 628, "泉州": 407, "邯郸": 203, "唐山": 638, "沧州": 367, "洛阳": 509, "南阳": 311, "襄阳": 206, "吕梁": 48, "大同": 159, "潍坊": 1041, "南通": 883, "衡阳": 420, "中山": 457, "宜昌": 319, "许昌": 151, "盐城": 488, "济宁": 534, "龙岩": 161, "上饶": 205, "马鞍山": 181, "赣州": 380, "金华": 424, "常德": 305, "延边": 102, "泰州": 172, "三亚": 460, "淮安": 368, "咸宁": 223, "无锡": 509, "常州": 420, "镇江": 506, "烟台": 976, "绍兴": 608, "莆田": 226, "东莞": 1270, "佛山": 894, "株洲": 274, "黄石": 170, "孝感": 233]
def zgNewDistMap = ["南京": 711, "大连": 143, "青岛": 1117, "济南": 577, "昆明": 726, "中山": 466, "佛山": 852, "盐城": 384, "北京": 563, "无锡": 342, "东莞": 428, "烟台": 853, "厦门": 289, "南通": 325, "太原": 310, "泉州": 354, "贵阳": 1324, "马鞍山": 141, "上海": 1776, "苏州": 385, "郑州": 610, "常州": 230, "洛阳": 228, "潍坊": 795, "赣州": 238, "淮安": 268, "深圳": 1034, "长沙": 1252, "石家庄": 296, "金华": 156, "南昌": 383, "泰州": 255, "广州": 1193, "三亚": 341, "宁波": 475, "唐山": 382, "镇江": 495, "济宁": 330, "莆田": 213, "沧州": 155, "宝鸡": 173, "重庆": 634, "沈阳": 494, "长春": 182, "襄阳": 164, "天津": 556, "合肥": 366, "宜昌": 252, "温州": 428, "南阳": 379, "成都": 2452, "绍兴": 147, "呼和浩特": 337, "武汉": 845, "西安": 1407, "邯郸": 220, "常德": 259, "杭州": 568, "黄石": 143, "大同": 151, "许昌": 289, "延边": 72, "咸宁": 230, "拉萨": 60, "株洲": 225, "上饶": 210, "孝感": 229, "吕梁": 78, "汕头": 329, "龙岩": 150, "衡阳": 328]
def fields = ["city","region","district_name","estate_type","area","avg_price","total_price","direction","fitment","room","hall","toilet","floor","height","build_structure","comm_id","dataType1","purpose","root","url", "p_date"]
def no_fields = ["build_date","build_type","floor_info"]
static main(args) {
def task = new MingShengStat();
// task.run()
// task.addNewLevel
// task.addHouseRate()
// task.houseFieldStat()
task.addDistBuildStr()
// task.statFloorInfo()
}
def addDistBuildStr(){
def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"]
def rows = []
for(def c : citys){
def row = []
row << c
def cityInfoDto = CommonDatautils.findCityInfo(c)
if(cityInfoDto){
def detailMap = cityInfoDto.district_detail
row << detailMap.finishBuildingDate.toDouble()*100
}
rows << row
}
ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计-竣工时间.xlsx", rows, "字段统计")
}
def houseFieldStat(){
def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"]
def rows = []
for(def c : citys){
def row = []
row << c
def total = findHouseCount(c)
for(def f : no_fields){
def num = findValueByCity(c, f);
if(num){
def rate = com.fangjia.script.common.DataHandleUtils.getDoubleRound(4, num*100/total)
row << rate
}else{
row << 0
}
}
rows << row
}
ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计2.xlsx", rows, "字段统计")
}
def statFloorInfo(){
def file = "D:\\数据\\民生\\字段覆盖率统计-楼层.xlsx"
ExcelUtils.multiThreadingModify(file,{d->
def total = findHouseCount(d['A'])
if(total>=100000){
total = 100000
}
def num = findHouseDatas(d['A']);
if(num){
def rate = com.fangjia.script.common.DataHandleUtils.getDoubleRound(4, num*100/total)
d['C'] = rate
}else{
d['C'] = 0
}
})
}
def addHouseRate(){
// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\估值数据统计结果20220816.xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\民生\\城市统计.xlsx",{d->
if(d['B']){
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
d['G'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, cityInfoDto.price_coverage*100)
def dists = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city","region","name"])
d['F'] = dists.size()
d['D'] = statNewDistCount(d['B'])
d['E'] = findHouseCount(d['B'])
}
})
}
def findValueByCity(city, field){
def count = 0
def sql = "select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell' and ${field} is not null"
ElasticUtils.executeSqlQuery(sql, {
count = it.count
println("$city\t${field} ${it.count}")
})
return count
}
def findHouseDatas(city){
def sql = "select floor_info from seawater where p_date >= 20200801 and city='${city}' and cat='sell' order by p_date desc limit 100000"
def f_count = 0
def count = 0
ElasticUtils.executeSqlQuery(sql, {
count++
def s = it.floor_info
if(s && !s.contains("暂")){
f_count = f_count +1
}
println("${count} $city\t floor_info ${s}")
})
return f_count
}
def findHouseCount(city){
def count = 0
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell'", {
count = it.count
println("$city\t${it.count}")
})
return count
}
def statNewDistCount(def city){
def sum = 0
if(ajkNewDistsMap.get(city)){
sum = sum + ajkNewDistsMap.get(city)
}
if(bkNewDistsMap.get(city)){
sum = sum + bkNewDistsMap.get(city)
}
if(ljNewDistMap.get(city)){
sum = sum + ljNewDistMap.get(city)
}
if(sfNewDistMap.get(city)){
sum = sum + sfNewDistMap.get(city)
}
if(chNewDistMap.get(city)){
sum = sum + chNewDistMap.get(city)
}
if(jwNewDistMap.get(city)){
sum = sum + jwNewDistMap.get(city)
}
if(zgNewDistMap.get(city)){
sum = sum + zgNewDistMap.get(city)
}
return sum
}
def addNewLevel(){
ExcelUtils.modify("D:\\数据\\民生\\城市统计.xlsx",{d->
if(d['B']){
if(cityLevel0.contains(d['B'])){
d['D'] = "一线"
}else if(cityLevel1.contains(d['B'])){
d['D'] = "新一线"
}else if(cityLevel2.contains(d['B'])){
d['D'] = "二线"
}else if(cityLevel3.contains(d['B'])){
d['D'] = "三线"
}else if(cityLevel4.contains(d['B'])){
d['D'] = "四线"
}else if(cityLevel5.contains(d['B'])){
d['D'] = "五线"
}
}
})
}
def run(){
def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"]
def rows = []
for(def c : citys){
def dbo = CommonDatautils.findAlCityDetail(c)
def row = []
if(dbo){
row << dbo.province
row << c
row << dbo.new_city_grade/1000
}else{
row << ""
row << c
row << ""
}
rows << row
}
ExcelUtils.write("D:\\数据\\民生\\城市统计.xlsx", rows, "数据")
}
}
@@ -0,0 +1,94 @@
package com.fangjia.script.gj.test
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.core.RegionData
import groovy.sql.Sql
def newRegionCodeFile = "D:\\数据\\建行\\二期\\行政区处理_20220923.xlsx"
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def specialCityMap = ["内蒙古##兴安":"兴安盟","四川##甘孜":"甘孜州","云南##文山":"文山州","青海##玉树":"玉树州"]
//ExcelUtils.modify(newRegionCodeFile,{d->
// if(d['A'] && !d['I']){
// def city = specialCityMap.get(d["A"]+"##"+d["B"])
// if(!city){
// city = RegionData.formatCity(d['B'])
// }
// if(city){
// def region = RegionData.formatRegion(city, d['E'])
// if(region){
// d['I'] = city
// d['J'] = region
// }
// }
// }
//})
def newDatas = ExcelUtils.read(newRegionCodeFile,0,1)
def newDataMap = [:]
for(def d : newDatas){
def key = d['I']+"##"+d['J']
def rets = newDataMap.get(key)
if(!rets){
newDataMap.put(key, d)
}
}
def ccbMap = getCcb92City(sqlInstance)
ExcelUtils.modify("D:\\数据\\亲邻\\region_code.xlsx",{d->
if(!d['C'] && d['D']){
def dbo = CommonDatautils.findAlCityDetail(d['A'])
if(dbo){
d['C'] = dbo.province
}
}
// if(!d['D']){
// def m = newDataMap.get(d['A']+"##"+d['B'])
// if(m){
// d['C'] = m['A']
// d['D'] = m['B']
// d['E'] = m['C']
// d['F'] = m['D']
// d['G'] = m['E']
// }
// def m = ccbMap.get(d['A']+"##"+d['B'])
// if(m){
// d['D'] = m.city
// d['E'] = m.cityId
// d['F'] = m.distId
// d['G'] = m.region
// }
// }
})
def getCcb92City(sqlInstance){
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj"
def map = [:]
sqlInstance.eachRow(sql, { r ->
def city = getCityName(r.CITYID)
def region = RegionData.formatRegion(city, r.DISTNAME)
def m = [:]
m.city = city
m.region = r.DISTNAME
m.cityId = r.CITYID
m.distId = r.DISTID
map.put(city+"##"+region, m)
})
return map;
}
def getCityName(cityId){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def city = null
cityCodeDataMap.each{cityName, id->
if(id == cityId){
city = cityName
}
}
return city;
}
@@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
@@ -15,10 +16,154 @@ import groovy.sql.Sql
import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.regex.Pattern
def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
DBUtils.seaweed.save(dist)
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
// def city = RegionData.formatCity(d['E'])
// if(city && !d['J']){
// def region = RegionData.formatRegion(city, d['G'])
// d['I'] = city
// d['J'] = region
// }
//})
//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d->
// def city = RegionData.formatCity(d['D'])
// if(!city){
// city = RegionData.formatCity(d['F'])
// }
// if(city && !d['H']){
// def region = RegionData.formatRegion(city, d['B'])
// d['G'] = city
// d['H'] = region
// }
//})
//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
//def fjwCitys = []
//for(def d : datas){
// fjwCitys << d['G']
//}
//def rows = []
//for(def d : datas){
// if(citys.contains(d['G'])){
// rows << d
// }
//}
//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域")
//for(def c : citys){
// if(!fjwCitys.contains(c)){
// println c
// }
//}
//
//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域")
//
//def map = [:]
//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx")
//for(def row : datas1){
// if(row['J']){
// def codeMap = [:]
// codeMap.prov = row['B']
// codeMap.city = row['F']
// codeMap.region = row['H']
// map.put(row['I']+"#"+row['J'], codeMap)
// }
//}
//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
//for(def row : datas2){
// if(row['H']){
// def codeMap = [:]
// codeMap.prov = row['E']
// codeMap.city = row['C']
// codeMap.region = row['A']
// map.put(row['G']+"#"+row['H'], codeMap)
// }
//}
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
// def key = d['B'] +"#"+d['C']
// def codeMap = map.get(key)
// if(codeMap && !d['D']){
// d['D'] = codeMap.prov
// d['E'] = codeMap.city
// d['F'] = codeMap.region
// }
//})
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
if(!d['D']){
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
if(infos && infos.size()==1){
def info = infos[0]
d['D'] = info.province_code
d['E'] = info.area_code
d['F'] = info.street_code
}
}
})
def findTownshipInfo(city, region, coll){
def q = new BasicDBObject()
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
def dbcursor = coll.find(q)
def rows = []
for(def d : dbcursor){
// if(d.street_name.contains(region)){
rows << d
// }
}
return rows
}
//def cityMap = CommonDatautils.findAllCityProvince()
//def rows = []
//rows << ["省", "市", "区域"]
//cityMap.each{c,p->
// def regions = CommonDatautils.getCityRegions(c)
// for(def r : regions){
// def row = []
// row << p
// row << c
// row << r
// rows << row
// }
//}
//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域")
//def q = new BasicDBObject()
//q.put("root", "anjuke.com")
//def dbcursor = coll.find(q)
//def map = [:]
//for(def d : dbcursor){
// map.put(d.city, d.city_short)
//}
//println map.size()
//println map
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
//DBUtils.seaweed.save(dist)
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
@@ -0,0 +1,142 @@
package com.fangjia.script.gj.test
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018)
DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018)
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//Thread.sleep(60*60*1000)
//while (true){
def q = new BasicDBObject()
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
q.put("status",new BasicDBObject('$ne', 1))
def dbcursor = school_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
JSON json = new JSONObject()
json.put("project", d.project)
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def jsonStr = d.map_info
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
if(jsonStr.contains("\n")){
jsonStr = jsonStr.replaceAll('\\n','')
}
result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
result.put("type", d.type)
json.put("result", result)
rets << json
if(rets.size()==5000){
println "共:"+rets.size()
send(rets, fixedThreadPool, school_coll)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
send(rets, fixedThreadPool, school_coll)
rets.clear()
}
//}
def send(def rets, def fixedThreadPool, def school_coll){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
//def rets = []
////def query = new BasicDBObject()
////query.put("source","搜学网")
//////query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
//////query.put("lat",null)
////def cursor = school_parse_coll.find(query)
////for(def o : cursor){
// def q = new BasicDBObject()
//// q.put("province","江苏省")
//// q.put("url",o.url)
// def dbcursor = sxue_school_coll.find(q)
// def index = 0
// for(def d : dbcursor){
// index++
// println index
// JSON json = new JSONObject()
// json.put("project", d.project)
// json.put("url", d.url)
//
// JSON result = new JSONObject()
// result.put("content", d.content)
// result.put("name", d.name)
// result.put("list_item", d.list_item)
//
// def jsonStr = d.map_info
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
// if(jsonStr.contains("\n")){
// jsonStr = jsonStr.replaceAll('\\n','')
// }
// result.put("json_data", jsonStr)
//
// result.put("type", d.type)
// json.put("result", result)
//
// rets << json
// if(rets.size()==5000){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
// }
// }
////}
//if(rets.size()>0){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
//}
@@ -647,9 +647,9 @@ class CZDataBatch extends BaseBatch{
}
}
if(!priceMap){
priceMap = findRegionRentPrice(priceMap, city, region, estate_type2)
}
// if(!priceMap){
// priceMap = findRegionRentPrice(priceMap, city, region, estate_type2)
// }
//5室及以上价格小于2000去除
if(priceMap){
@@ -0,0 +1,39 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ElasticUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* ²¹³äѧУ¾­Î³È
* @author shenjian
* @since 2022/9/8
*/
class FillLngLat {
static ExecutorService executorService = Executors.newFixedThreadPool(4)
static void main(String[] args) {
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
final def data = dataT
executorService.submit(new Runnable() {
@Override
void run() {
println(data)
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
if (lngLat != null && lngLat.size() == 2 ) {
JSONObject jsonObject = new JSONObject()
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
}
}
})
})
executorService.shutdown()
}
}
@@ -0,0 +1,102 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.RegionData
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.poi.entity.Community
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import org.apache.commons.collections.CollectionUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.stream.Collectors
/**
* 逆解析第三方小区经纬度
*/
class ReGeoFromCommunity {
static final ExecutorService executorService = Executors.newFixedThreadPool(5)
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
Map<String, String> map = new ConcurrentHashMap<>()
def rets = this.findCommunities()
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
final BulkRequest bulkRequest = new BulkRequest();
for (def retTemp : rets) {
final def ret = retTemp
executorService.submit(new Runnable() {
@Override
void run() {
try {
final String id = ret._id // 索引ID,并非小区ID
if (ret.location) {
String lon = ret.location.lon
String lat = ret.location.lat
def addressComponent = GaoDeUtils.reGeoLocation(lon + "," + lat)
if (addressComponent) {
def city_format = RegionData.formatCity(addressComponent.city)
def region_format = RegionData.formatRegion(city_format, addressComponent.district)
def township = addressComponent.township
Map<String, String> obj = new HashMap<>()
obj.put("city", city_format)
obj.put("region", region_format)
obj.put("township", township)
Map<String, Object> doc = new HashMap<>();
doc.put("address_component", obj)
bulkRequest.add(new UpdateRequest("community", id).doc(doc));
}
}
synchronized (map) {
map.put(id, id)
if (map.size() % 1000 == 0) {
atomicLong.addAndGet(map.size())
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
bulkRequest = new BulkRequest();
map.clear()
println(atomicLong.intValue())
}
}
} catch(Exception ex) {
} finally {
countDownLatch.countDown()
}
}
})
};
countDownLatch.await();
if (map.size() > 0) {
atomicLong.addAndGet(map.size())
EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT)
bulkRequest = new BulkRequest();
map.clear()
println(atomicLong.intValue())
}
System.exit(1)
}
static def findCommunities(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.mustNot(QueryBuilders.existsQuery("address_component"))
.must(QueryBuilders.existsQuery("location"))
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class, 200000)
return communities
}
}
@@ -10,6 +10,7 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@@ -27,15 +28,15 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.ShenJian)
class SeaweedAroundPoi extends BaseBatch {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.235", 27018)
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
//static ExecutorService executorService = Executors.newFixedThreadPool(2)
static void main(String[] args) {
def types = listAllType()
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
//executorService.shutdown()
}
/**
@@ -64,33 +65,40 @@ class SeaweedAroundPoi extends BaseBatch {
}
static dealPoi(DBObject query, def types) {
query.put("status", 1)
conn136.find(query).batchSize(5000).each {objTemp ->
query.put("canyin_status", 0)
conn136.find(query).batchSize(5000).limit(20000).noCursorTimeout(true).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
// executorService.submit(new Runnable() {
// @Override
// void run() {
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
if (obj.lng) {
def lng = SeaweedUtils.formatLatLng(obj.lng)
def lat = SeaweedUtils.formatLatLng(obj.lat)
// {'lon': 121.810877, 'lat': 31.737642}
// def lng = 121.810877
// def lat = 31.737642
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
println("已处理小区:" + obj)
}
})
}
// })
// }
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("city", new BasicDBObject('$in', ["连云港", "南通", "苏州", "无锡", "盐城", "扬州"]))
//query.put("city", "连云港")
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
@@ -163,17 +171,18 @@ class SeaweedAroundPoi extends BaseBatch {
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
{
put("餐饮", "05");
// put("餐饮", "05");
// put("地铁", "150500");
// put("公交", "1507");
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
// put("写字楼", "120201");
// put("学校", "141200|141201|141202|141203|141204");
// put("银行", "1601|1603");
// put("大型超市", "0604");
put("大型超市", "0604");
// put("商场", "0601");
// put("餐饮", "0501|0502|0503|0504");
// put("旅游景点", "110000");
put("公园广场", "110100|110101|110102|110103|110104|110105")
// put("图书馆", "140500");
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
// put("丧葬设施", "071900|071901|071902|071903")
@@ -0,0 +1,84 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边POI数据获取
*
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundPoiFromExcel {
static void main(String[] args) {
def types = listAllType()
dealPoi(new BasicDBObject(), types)
}
static dealPoi(DBObject query, def types) {
com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item ->
def city = item['H']
def lng = item['AQ']
def lat = item['AR']
if (lng) {
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr)
}
Thread.sleep(1000)
print(lng + " " + lat)
}
});
};
public static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
};
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
{
// put("餐饮", "05");
// put("地铁", "150500");
// put("公交", "1507");
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
// put("写字楼", "120201");
// put("学校", "141200|141201|141202|141203|141204");
// put("银行", "1601|1603");
put("大型超市", "0604");
// put("商场", "0601");
// put("餐饮", "0501|0502|0503|0504");
// put("旅游景点", "110000");
put("公园广场", "110100|110101|110102|110103|110104|110105")
// put("图书馆", "140500");
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
// put("丧葬设施", "071900|071901|071902|071903")
}
}
}
@@ -1,6 +1,6 @@
package com.fangjia.script.temp.shenjian.poi.entity;
package com.fangjia.script.temp.shenjian.poi.entity
public class Community extends MyId {
class Community extends MyLocation {
private String id;
private String city;
private String region;
@@ -0,0 +1,16 @@
package com.fangjia.script.temp.shenjian.poi.entity
import org.elasticsearch.common.geo.GeoPoint
class MyLocation extends MyId {
GeoPoint location;
GeoPoint getLocation() {
return location
}
void setLocation(GeoPoint location) {
this.location = location
}
}
@@ -3,12 +3,14 @@ package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import com.fangjia.script.temp.shenjian.poi.entity.MyId
import com.fangjia.script.temp.shenjian.poi.entity.MyLocation
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.search.*
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.geo.GeoPoint
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
@@ -23,7 +25,7 @@ import org.slf4j.LoggerFactory
*/
class EsUtils {
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
public static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
/**
@@ -35,7 +37,7 @@ class EsUtils {
* @param <T>
* @return
*/
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
public static <T extends MyLocation> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
List<T> result = new ArrayList<>();
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
@@ -43,7 +45,11 @@ class EsUtils {
searchSourceBuilder.query(boolQuery);
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
//searchSourceBuilder.size(1);
searchSourceBuilder.size(10000);
if (limitSize < 10000) {
searchSourceBuilder.size(limitSize)
} else {
searchSourceBuilder.size(10000)
}
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
SearchResponse searchResponse = null;
try {
@@ -57,6 +63,13 @@ class EsUtils {
for (SearchHit searchHit : searchHits) {
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
t.set_id(searchHit.getId())
Map<String, Double> location = searchHit.getSourceAsMap().get("location")
if (location != null) {
Double lon = location["lon"]
Double lat = location["lat"]
GeoPoint geoPoint = new GeoPoint(lat, lon)
t.setLocation(geoPoint)
}
result.add(t);
}
}
@@ -146,11 +146,15 @@ class GaoDeUtils {
while (result.size() < count) {
params.page = "${page}".toString()
page++
params.key = this.getGaodeKey("v5/place/around")
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
params.key = this.getGaodeKey("v3/place/around")
//params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
params.key = this.getGaodeKey("v5/place/around")
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
params.key = this.getGaodeKey("v3/place/around")
//params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
@@ -258,7 +262,7 @@ class GaoDeUtils {
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoiDetail.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
@@ -334,10 +338,6 @@ class GaoDeUtils {
/**
* 经纬度转地址<br>
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
* @author yinjihuan
* @date 2016Äê9ÔÂ9ÈÕ
* @param location
* @return
*/
static def getGaoDeAddress(location) {
Map<String, String> params = new HashMap<String, String>()
@@ -369,6 +369,41 @@ class GaoDeUtils {
return null
}
/**
* 地址逆解析
* @param location
* @return 经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
*/
static def reGeoLocation(location) {
Map<String, String> params = new HashMap<String, String>()
params.put("location", location)
String content
try {
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
params.put("key", this.getGaodeKey("v3/geocode/regeo"))
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content)
Map<String, Object> infoMap = DataHandleUtils.JsonToMap(json);
if (infoMap && infoMap.info == "OK") {
def regeocode = infoMap.regeocode
if (regeocode) {
return regeocode.addressComponent
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return null
}
/**
* 获取地址经纬度
*/
@@ -1,5 +1,6 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{
}
def rowIndex = 0;
DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018)
DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018)
static main(args) {
runBatch(new DataStandardBatch(), args)
}
@@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias->
// if(city == "北京"){
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
}
while(flag){
@@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
district = null;
}else{
district = getMatchNameFromMap(data.city, data.region, data.name, data.address)
district = getMatchNameFromMap(data)
if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区”
district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address)
def copyObj = data
copyObj.name = data.name+"小区"
district = getMatchNameFromMap(copyObj)
}
if(!district){
def newName = filterSpacialChar(data.name);//包含特殊符号
if(newName) {
district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address)
def copyObj = data
copyObj.name = newName.replaceAll("_", "")
district = getMatchNameFromMap(copyObj)
if(!district){
def s1 = newName.substring(0, newName.indexOf("_"))
if (s1) {
//特殊符号前的内容匹配
//district = getMatchName(data.city, data.region, s1, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, s1, data.address)
copyObj.name = s1
district = getMatchNameFromMap(copyObj)
if (!district) {
def s2 = newName.replace(s1, "").replaceAll("_", "")
def newName2 = null;
if (s2 && s2.length() >= 3) {
//特殊符号后的内容匹配
// district = getMatchName(data.city, data.region, s2, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, s2, data.address)
copyObj.name = s2
district = getMatchNameFromMap(copyObj)
if (district) {
type = 3;
println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]"
}
} else {
newName2 = newName.replaceAll("_", "")
copyObj.name = newName2
//district = getMatchName(data.city, data.region, newName2, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, newName2, data.address)
district = getMatchNameFromMap(copyObj)
if (district) {
type = 4;
println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]"
@@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{
}
if (!district) {
district = getMatchNameFromMap(data.city, null, newName2, data.address)
copyObj.name = newName2
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if(!district){
district = getMatchNameFromMap(data.city, null, s1, data.address)
copyObj.name = s1
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if (district) {
type = 5;
} else {
if (s2 && s2.length() >= 3) {
district = getMatchNameFromMap(data.city, null, s2, data.address)
copyObj.name = s2
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if (district) {
type = 6;
}
@@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{
}catch (Exception e){
addr = data.address;
}
district = getMatchAddressFromMap(data.city, data.region, addr, data.name)
def copyObj = data
copyObj.address = addr
district = getMatchAddressFromMap(copyObj)
if(district){
type = 8;
}else{
district = getMatchAddressFromMap(data.city, null, addr, data.name)
copyObj.region = null
district = getMatchAddressFromMap(copyObj)
if(district){
type = 9;
}
@@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{
if(!district){
def districts = getMatchPingYing(data.city, data.region, data.name, addr);
district = checkMatchAddress(districts, addr);
district = checkMatchAddress(districts, addr, data);
if(district){
type = 10;
}else if(districts.size()==1){
district = districts[0];
type = 14
district = getFilerMatchDistrict(districts[0], data);
if(district){
type = 14
}
}
}
if(!district){
district = getOpenApiMatch(data.city, data.region, data.name, addr);
def copyObj = data
copyObj.address = addr
district = getOpenApiMatch(copyObj)
if(!district && addr){
district = getOpenApiMatch(data.city, data.region, addr, data.name);
district = getOpenApiMatch(copyObj)
}
if(district){
if(data.region != district.region){
@@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{
return null;
}
def checkMatchAddress(def districts, def address){
def checkMatchAddress(def districts, def address, def data){
def district = null;
if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){
def tempDist = districts[0];
@@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(district){
return getFilerMatchDistrict(district, data)
}
return district;
}
@@ -584,54 +611,154 @@ class DataStandardBatch extends BaseBatch{
}*/
def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName);
def getMatchNameFromMap(def data){
//查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price
def dbo = findSameNameFromMap(data.city, data.region, data.name);
def addrFlag = null
if(address){
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(data.address){
addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
}
if(!dbo){
//别名
def distrs = findAliasNameFromMap(city, region, disName);
def distrs = findAliasNameFromMap(data.city, data.region, data.name);
if(distrs && distrs.size()==1){
dbo = distrs[0]
}else if(distrs && distrs.size()>1 && address){
}else if(distrs && distrs.size()>1 && data.address){
for(def d : distrs){
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){
dbo = d;
break;
}
}
}
if(dbo){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
//地址
if(!dbo){
def result = findAddressFromMap(city, region, disName)
def result = findAddressFromMap(data.city, data.region, data.name)
if(result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && address){
}else if(result && result.size()>1 && data.address){
for(def d : result){
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){
dbo = d;
break;
}
}
}
if(dbo){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
}
}
if(dbo){
return getFilerMatchDistrict(dbo, data)
}
return dbo;
}
def getFilerMatchDistrict(dbo, data){
def types = []
if(data.estate_type){
types = getEstateTypes(data.estate_type)
}
if(types.size()>0 && types.size() == 1){
if(types.contains(dbo.estate_type)){
return dbo;
}else{
def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id)
if(!ret){
data.put("msg","物业类型不匹配")
THIRD_MATCH_FILTER_COLL.save(data)
}
return null
}
}else{
def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger())
def price = priceMap.get(dbo.estate_type)
if(price && data.avg_price && data.avg_price.toString().isNumber()){
def pre = (data.avg_price-price)*100/price
if(pre>=-20 && pre<=20){
return dbo;
}else{
def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
if(!ret){
data.put("msg","房源价格过滤")
THIRD_PRICE_FILTER_COLL.save(data)
}
return null
}
}
}
return dbo
}
def findHousePrice(def city, def region, def name, def startDate){
def typePriceMap = [:]
try{
String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'"
def houseMap = [:]
ElasticUtils.executeSqlQuery(sql,{ data->
if(data.estate_type){
def key = ""
if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){
key = "普通住宅"
}else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){
key = "商住"
}else if( data.estate_type.contains("别墅")){
key = "别墅"
}else{
key = "其它"
}
def houses = null
if(key){
houses = houseMap.get(key)
if(!houses){
houses = []
}
}
if(data.avg_price){
houses << data.avg_price
houseMap.put(key, houses)
}
}
})
if(houseMap){
houseMap.each{k,v->
if(v && v.size()>=3){
def price = (int)(v.sum()/v.size())
typePriceMap.put(k, price)
}
}
}
}catch(Exception e){
e.printStackTrace()
}
return typePriceMap
}
def getEstateTypes(str){
def types = []
if(str){
if(str.contains("住宅") || str.contains("公寓")){
types << "普通住宅"
}
if(str.contains("商住") || str.contains("商务")){
types << "商住"
}
if(str.contains("别墅")){
types << "别墅"
}
}
return types;
}
/*def getMatchAddress(def city, def region, def address, def name, def cat){
def dbo = CommonDatautils.findSeaweed(city, region, address, cat);
@@ -664,22 +791,23 @@ class DataStandardBatch extends BaseBatch{
}
return dbo;
}*/
def getMatchAddressFromMap(def city, def region, def address, def name){
def getMatchAddressFromMap(def copyObj){
def dbo = null
if(address && isStandardRoad(address)){
dbo = findSameNameFromMap(city, region, address);
def name = copyObj.name
if(copyObj.address && isStandardRoad(copyObj.address)){
dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address);
}
if(!dbo){
def result = null
if(address && isStandardRoad(address)){
result = findAddressFromMap(city, region, address);
if(copyObj.address && isStandardRoad(copyObj.address)){
result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address);
}
if(result && result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
}else if(result && result.size()>1 && copyObj.name){
def score = null;
for(def d : result){
if(d.name == name){
if(d.name == copyObj.name){
dbo = d;
break;
}else{
@@ -706,7 +834,7 @@ class DataStandardBatch extends BaseBatch{
}
}
if(!dbo){
result = findAddrAliasFromMap(city, region, address);
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
if(result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
@@ -718,6 +846,11 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(dbo){
return getFilerMatchDistrict(dbo, copyObj)
}
}
return dbo;
}
@@ -756,7 +889,7 @@ class DataStandardBatch extends BaseBatch{
if(arrays[0] && arrays.size()==2){
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ")
if(py1 == py2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}
}
}
@@ -768,11 +901,15 @@ class DataStandardBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name, def address){
if(StringUtils.isNotEmpty(name)){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false)
def getOpenApiMatch(def data){
if(StringUtils.isNotEmpty(data.name)){
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit]
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
if(dbo){
return getFilerMatchDistrict(dbo, data)
}
return dbo
}
}
return null;
@@ -783,6 +920,9 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","北京")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -948,9 +1088,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays[0] && arrays[0]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
@@ -962,9 +1102,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays[0] && arrays[0]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -990,18 +1130,18 @@ class DataStandardBatch extends BaseBatch{
def alias = v.alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}else if(v && v.mapping_alias){
def mappingAlias = v.mapping_alias
if(mappingAlias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1016,18 +1156,18 @@ class DataStandardBatch extends BaseBatch{
def alias = v.alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}else if(v && v.mapping_alias){
def mappingAlias = v.mapping_alias
if(mappingAlias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1052,9 +1192,9 @@ class DataStandardBatch extends BaseBatch{
def alias = v.addr_alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1068,9 +1208,9 @@ class DataStandardBatch extends BaseBatch{
def alias = v.addr_alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1093,9 +1233,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays && arrays.size()==2 && arrays[1]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1106,9 +1246,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays && arrays.size()==2 && arrays[1]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "城市房产"//
def citys = findCrawlCity(source, 202205);
def source = "°²¾Ó¿Í"//
def citys = findCrawlCity(source, 202208);
for(def city : citys){
def count1 = findCount(source, city, 202205);
def count2 = findCount(source, city, 202206);
def count1 = findCount(source, city, 202208);
def count2 = findCount(source, city, 202209);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -1,5 +1,8 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
@@ -11,31 +14,38 @@ import org.bson.types.ObjectId
* 福州农商行
* 将抓取的小区均价同步到第三方均价表中
*/
class SynchDistrictPrice {
@BatchOwner(Watcher.GaoJuan)
class SynchDistrictPrice extends BaseBatch{
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
//"fang.com":"搜房",
// def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
//,"城市房产":"城市房产"
def esSources = ["365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","吉屋":"吉屋"]//
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
def esSources = ["中原地产":"中原地产"]
static main(args) {
def task = new SynchDistrictPrice()
// task.run()
task.run1()
runBatch(new SynchDistrictPrice(), args);
}
@Override
public void process(Object args) {
run1()
}
def run1(){
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
esSources.each{esSource, mgSource->
for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource)
def rets = findAllEsData(city, esSource, mgSource, mdate)
println city +" "+rets.size()
synchData(city, rets, mgSource)
}
@@ -43,11 +53,11 @@ class SynchDistrictPrice {
}
def findAllEsData(city,esSource, mgSource){
def findAllEsData(city,esSource, mgSource, mdate){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-08-01'", {d->
def s = d.city +"#"+d.region+"#"+d.name
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
if(!names.contains(s)){
names << s
def data = new BasicDBObject()
@@ -57,6 +67,7 @@ class SynchDistrictPrice {
data.put("address", d.address)
data.put("avg_price", d.avg_price)
data.put("source", mgSource)
data.put("estate_type",d.property_type)
data.put("s_date", SHORT_DATE)
data.put("url", d.url)
data.put("c_date", new Date())
@@ -134,7 +145,7 @@ class SynchDistrictPrice {
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
}else{
for(def r : rets){
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source)
if(obj){
updateIndex++
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
@@ -142,6 +153,7 @@ class SynchDistrictPrice {
def udata = new BasicDBObject()
udata.put("avg_price", r.avg_price)
udata.put("estate_type", r.estate_type)
udata.put("m_date",new Date())
thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata))
@@ -171,13 +183,16 @@ class SynchDistrictPrice {
}
def findIsExists(def city, def region, def name, def sDate, def source){
def findIsExists(def city, def region, def name, estate_type, def sDate, def source){
def q = new BasicDBObject()
q.put("source", source)
q.put("s_date", sDate)
q.put("city", city)
q.put("region", region)
q.put("name", name)
if(estate_type){
q.put("estate_type",estate_type)
}
def obj = thirdPartyColl.findOne(q)
return obj
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202206);
def count2 = findTotalCount(source, 202207);
def count1 = findTotalCount(source, 202208);
def count2 = findTotalCount(source, 202209);
println source +" "+count1 +" "+count2
}
}
@@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
def cityIndex = 0;
@@ -354,12 +354,13 @@ public class OpenApiClient {
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
// params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
//192.168.0.100:30115
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
@@ -369,7 +370,7 @@ public class OpenApiClient {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
println("[http error] ${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde"
def static GAO_DE_AK = "90010d53a16ce9df544acab297645106"
/**
* 冪帠僅蛌華硊<br>
@@ -38,8 +38,8 @@ class BasicInfoDataSynch extends BaseBatch{
synchRegion()
synchTown()
synchDict()
synchDistrict("district")
synchDistrict("office")
// synchDistrict("district")
// synchDistrict("office")
}
def synchCitys(){
@@ -299,7 +299,7 @@ class BasicInfoDataSynch extends BaseBatch{
}
def synchDistrictByCity(fjCity, districts, cat){
def zsDistrictCursor = ZS_SEAWEED_CONN.find(new BasicDBObject("city", fjCity).append("cat", cat))
def zsDistrictCursor = ZS_SEAWEED_CONN.find(new BasicDBObject("city", fjCity).append("category", cat))
def zsDistricts = []
def zsDistrictMap = [:]
def fjDistrictMap = [:]
@@ -322,7 +322,6 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("estate_type",fjDistrict.estate_type2)
udata.put("township",fjDistrict.township)
udata.put("alias",fjDistrict.alias)
udata.put("cat",cat)
udata.put("update_date",new Date().getTime())
if(fjDistrict.addr_alias){
udata.put("addr_alias",fjDistrict.addr_alias)
@@ -50,6 +50,21 @@ public class AmapPoiDetail {
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@com.alibaba.fastjson.annotation.JSONField(name = "business")
private BusinessDTO business;
@lombok.NoArgsConstructor
@lombok.Data
public static class BusinessDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "business_area")
private String businessArea;
@com.alibaba.fastjson.annotation.JSONField(name = "keytag")
private String keytag;
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_today")
private String opentimeToday;
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_week")
private String opentimeWeek;
}
@lombok.NoArgsConstructor
@lombok.Data
@@ -64,7 +79,7 @@ public class AmapPoiDetail {
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
private String title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}