抓无锡村庄

This commit is contained in:
2021-07-22 09:59:18 +08:00
parent 7dff8154c5
commit ed0f3a05d4
13 changed files with 772 additions and 623 deletions
@@ -258,8 +258,9 @@ public class BaiDuApiUtils {
params.put("coords", lng.toString() + "," + lat.toString());
params.put("from", String.valueOf(from));
HttpRequest request = new HttpRequest();
String result = null;
try{
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
changeAK();
@@ -0,0 +1,88 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 建行模型估值项目
* 小区建筑类型数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
runBatch(new BigdataCommbuildtypeBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rets = []
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"])
def rows = createDatas(cityCodeDatas, distCodeMap, dists)
rets.addAll(rows)
println city +" "+rows.size()
}
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false);
println "txt 生成"
}
def createDatas(cityCodeDatas, distCodeMap, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, distCodeMap, dist)
if(ret){
rets << ret
}
println dist.city +" "+dist.region+" "+dist.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(cityCodeDatas, distCodeMap, dist){
def ret = []
ret << cityCodeDatas.city_code
ret << dist._id.toString()
ret << CcbCodeUtils.parseBuildType(dist.build_type)
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
}
@@ -49,7 +49,7 @@ class BigdataCommunityBatch extends BaseBatch{
rows.addAll(rets)
println city +" "+rets.size()
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def allDatas = []
@@ -61,16 +61,20 @@ class BigdataCommunityBatch extends BaseBatch{
}
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
def rets = []
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
def dist = dists[i]
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets << createData(cityCodeDatas, distCodeMap, dist)
def ret = createData(cityCodeDatas, distCodeMap, dist)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -85,70 +89,76 @@ class BigdataCommunityBatch extends BaseBatch{
def createData(def cityCodeDatas, def distCodeMap, def dist){
def ret = []
ret << cityCodeDatas.city_code
if(dist.region){
ret << distCodeMap.get(dist.region)
}else{
ret << ""
}
try{
ret << cityCodeDatas.city_code
if(dist.region){
ret << distCodeMap.get(dist.region)
}else{
ret << ""
}
ret << dist._id.toString()
ret << dist.region
ret << dist.name
ret << dist.address
if(dist.lat){
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
}else{
ret << ""
ret << ""
}
ret << dist._id.toString()
ret << dist.region
ret << dist.name
ret << dist.address
if(dist.lat){
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
}else{
ret << ""
ret << ""
}
ret << dist.all_area
ret << dist.build_area
ret << dist.virescence_percent
ret << dist.cubage_percent
ret << dist.car_site_count
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "物业小区"
ret << ""
if(dist.address){
if(dist.address.toString().endsWith("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << number
ret << dist.all_area
ret << dist.build_area
ret << dist.virescence_percent
ret << dist.cubage_percent
ret << dist.car_site_count
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "物业小区"
ret << ""
if(dist.address){
if(dist.address.toString().endsWith("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << number
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
ret << ""
ret << ""
}
}else{
ret << 0
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << ""
ret << ""
ret << ""
}catch(Exception e){
println dist.city + " "+dist.name
e.printStackTrace()
}
ret << 0
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << ""
ret << ""
ret << ""
return ret;
}
}
@@ -0,0 +1,416 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 成交数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static main(args) {
runBatch(new BigdataDealBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def trades = getCityData(city)
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other")
// rows.addAll(rets)
def youDataTrade = getYouDaData()
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rets)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false);
println "txt 生成"
def rets = []
rets << headers
rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
def result = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(source == "youda"){
result << converYouDaData(trade, cityCodeDatas, distCodeMap)
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}else{
result << converData(trade, cityCodeDatas, distCodeMap)
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def obj = CommonDatautils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.city_code
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
if(trade.fjRegion){
ret << distCodeMap.get(trade.fjRegion)
}else{
ret << ""
}
ret << trade.region
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().endsWith("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().endsWith("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss")
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
ret << cityCodeDatas.city_code
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
if(trade.region){
ret << distCodeMap.get(trade.region)
}else{
ret << ""
}
ret << trade.region
if(trade.address){
if(trade.address.toString().endsWith("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
ret << number
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
ret << ""
}
ret << trade.dist_id.toString()
ret << trade.master_name
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
ret << ""//期数
ret << trade.floor
ret << ""
ret << trade.height
ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.avg_price
ret << trade.total_price
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss")
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
def root = sourceMap.get(trade.source)
if(root){
if(CcbCommonUtils.rootMap.get(root)){
ret << CcbCommonUtils.rootMap.get(root)
}else{
ret << "99"
}
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.total_price
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def getHouseRoom(def room, def hall, def toilet){
def s = ""
if(room){
s = s + room +"室"
}
if(hall){
s = s + hall +"厅"
}
if(toilet){
s = s + toilet +"卫"
}
return s;
}
def getCityData(def city){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date()))
if(trades){
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return tradesList
}
def getYouDaData(){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate));
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
rets << obj;
}
}
return rets;
}
}
@@ -34,13 +34,14 @@ class BigdataDistrictBatch extends BaseBatch{
}
def allDatas = createDatas(rows)
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8");
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8",false);
}
def createDatas(rows){
@@ -76,6 +77,9 @@ class BigdataDistrictBatch extends BaseBatch{
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
def lists = []
for(def db : dbcursor){
if(db.fangjia_district && db.fangjia_district=="二道"){
println db.fangjia_district+" "+city
}
if(db.district_code){
def m = [:]
m.id = db._id.toString()
@@ -54,7 +54,7 @@ class BigdataForsaleBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
@@ -125,10 +125,11 @@ class BigdataForsaleBatch extends BaseBatch{
}
latch.await()
println city + " 房源总量:"+retMap.size()
def allDatas = createSellDatas(city, retMap.values());
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true);
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true);
println "txt文件生成 "
}
@@ -340,8 +341,8 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
if(rootMap.get(seawater.root)){
rowMap.root = rootMap.get(seawater.root)
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
@@ -1,521 +0,0 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 出售房源数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataForsaleBatch2 extends BaseBatch{
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
static main(args) {
runBatch(new BigdataForsaleBatch2(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def distMap = getDistrictMap(city)
exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731);
}
}
def getDistrictMap(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
if(d.alias){
d.alias.each{name->
if(!distMap.containsKey(d.region+"##"+name)){
distMap.put(d.region+"##"+name, d)
}
}
}
}
return distMap;
}
def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){
println city +" 小区key:"+distMap.size()
def cityCode = cityCodeDatas.city_code
def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap)
def allDatas = createSellDatas(city, retMap);
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true);
println "txt文件生成 "
}
def createSellDatas(def city, def map){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(map.size())
println city +"总房源量:"+map.size()
def datas = []
map.each{k,m->
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
datas << converData(m)
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return datas ;
}
def converData(def m){
def data = []
data << m.city_code
data << m.houseId
data << ""
data << m.city
data << m.region_code
data << m.region
data << m.district_id
data << m.district_name
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.address==null?"":m.address==null
data << m.build_year==null?"":m.build_year
data << m.proptype==null?"":m.proptype
data << m.propyears==null?"":m.propyears
data << m.avg_price==null?"":m.avg_price
data << m.total_price==null?"":m.total_price
data << ""
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.house_structure_code==null?"":m.house_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
data << m.house_type==null?"":m.house_type
data << m.room==null?"":m.room
data << m.hall==null?"":m.hall
data << ""
data << m.toilet==null?"":m.toilet
data << m.direction_code==null?"":m.direction_code
data << m.fitment_code==null?"":m.fitment_code
data << ""
data << ""
data << m.p_date==null?"":m.p_date
data << ""
data << ""
data << m.root
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.c_date==null?"":m.c_date
data << m.m_date==null?"":m.m_date
data << m.source
data << ""
return data
}
def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def retMap = new ConcurrentHashMap()
println city +" 没过滤前:"+ houses.size()
if(houses){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(houses.size())
for (int k=0;k<houses.size();k++) {
final def map = houses[k];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def rowMap = converData(distMap, map, cat)
if(retMap){
retMap.put(rowMap.houseId, rowMap)
println totalCount.get() +" [匹配success]"+city+" "+m.district_name+" "+new Date().toLocaleString()
}else{
println totalCount.get() +" [匹配fail]"+city+" "+m.district_name+" "+new Date().toLocaleString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
}
latch.await()
return retMap;
}
def converData(distMap, map, cat){
def rMap = distMap.get(map.region +"##"+map.district_name)
if(!rMap){
return null
}
DBObject seawater = new BasicDBObject(map)
if(!seawater.total_price){
return null
}
if(!seawater.avg_price){
return null
}
if(!seawater.height){
seawater.height = null
}
if(seawater.height && seawater.height<1){
return null
}
if(seawater.height && seawater.height>60){
return null
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=5 || seawater.room<1){
return null
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
return null
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
return null
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
return null
}
if(seawater.avg_price && seawater.avg_price<10){
return null
}
}
rowMap.houseId = seawater.id
rowMap.city = city
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
if(rootMap.get(seawater.root)){
rowMap.root = rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
}
def regionCode = distCodeMap.get(rMap.region)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
rowMap.city_code = cityCode
rowMap.region = rMap.region
rowMap.region_code = regionCode
rowMap.district_id = rMap.get("_id").toString()
rowMap.district_name = rMap.name
rowMap.build_year = buildYear;
rowMap.proptype = housePro
rowMap.propyears = propertyYear;
rowMap.estate_type = estateType
rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
rowMap.address = rMap.address
rowMap.build_structure_code = struCode
rowMap.house_structure_code = null//getHouseStructureCode();
rowMap.source = "FJW"
return rowMap
}
def getPublicDate(def pDate){
if(pDate>=20181001 && pDate<20200201){
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
}
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
return pdate;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
private String csvHandlerStr(String str) {
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
String tempDescription=str;
//如果有逗号
if(str.contains(",")||str.contains(",")){
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
if(str.contains("\"")){
// tempDescription=str.replace("\"", "\"\"");
tempDescription=str.replace("\"", "");
}
//在将逗号转义
tempDescription="\""+tempDescription+"\"";
}
return tempDescription;
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
/*
* 生成一个用不重复的ID
*/
def GenNonDuplicateID(randomLength){
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
}
def marge(){
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
def ids = new HashSet();
def index = 0
file.eachLine{it->
index++
if(index>1){
def arrs = it.split("\\|@\\|")
def id = arrs[1]
if(!ids.contains(id)){
ids << id
}
}
}
println index
println ids.size()
}
}
@@ -7,8 +7,8 @@ import com.mongodb.DBCollection
class CcbCommonUtils {
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
// public static def citys = ["北京"]
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -18,6 +18,11 @@ class CcbCommonUtils {
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def findCityCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
@@ -58,4 +63,46 @@ class CcbCommonUtils {
}
return rets
}
public static def findTradesDetail(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
}
return lists
}
}
@@ -52,7 +52,7 @@ class CityBatch extends BaseBatch{
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8");
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false);
}
def createDatas(rows){
@@ -0,0 +1,21 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys) {
def data = CcbCommonUtils.findCityCodeValue(city)
def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
def count = 0
sqlInstance.execute(sql,{
count = it.count
});
println city +" "+count
}
@@ -74,20 +74,20 @@ class AddThirdPriceDistrict {
static main(args) {
def task = new AddThirdPriceDistrict();
// task.run()
task.run()
def mapName = "锦华星光苑3,4幢"
// def mapName = "锦华星光苑3,4幢"
//// println task.pattern("港新花苑A1区");
println mapName.replaceAll("\\w+-?\\w+(幢)", "")
println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
// println mapName.replaceAll("\\w+-?\\w+(幢)", "")
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
}
def run(){
// exportDatas();
exportDatas();
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// addInfo();
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
}
def addCrawlMapInfo( def path){
@@ -135,29 +135,32 @@ class AddThirdPriceDistrict {
def exportDatas(){
def city = "无锡"
def datas = findMatchErrorDatas(city);
println "共【"+datas.size()+"】条"
datas = filterName(datas);
println "过滤后共【"+datas.size()+"】条"
def resMap = groupByName(datas);
def rets = []
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
resMap.each {k,v->
def ret = []
def arrs = k.split("##")
ret << arrs[0]
ret << arrs[1]
if(arrs.size()==3 && arrs[2]){
ret << arrs[2]
}else{
ret << ""
//跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞",
def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
for(def city : citys){
def datas = findMatchErrorDatas(city);
println "共【"+datas.size()+"】条"
datas = filterName(datas);
println "过滤后共【"+datas.size()+"】条"
def resMap = groupByName(datas);
def rets = []
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
resMap.each {k,v->
def ret = []
def arrs = k.split("##")
ret << arrs[0]
ret << arrs[1]
if(arrs.size()==3 && arrs[2]){
ret << arrs[2]
}else{
ret << ""
}
ret << resAddrMap.get(k)
ret << v.join(",")
rets << ret
}
ret << resAddrMap.get(k)
ret << v.join(",")
rets << ret
ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后")
}
ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后")
}
def filterName(datas){
@@ -465,7 +468,7 @@ class AddThirdPriceDistrict {
def findMatchErrorDatas(def city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date",202005)
q.put("s_date",new BasicDBObject('$gte', 202101))
// q.put("name","博济聚创科技园")
// q.put("source","贝壳找房")
def dbcursor = matchErrorLogColl.find(q)
@@ -56,11 +56,11 @@ class GaoDeMapCrawl extends BaseBatch{
// def prefix = "road_"
// def categorys = ["190301"]//道路相关
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
def prefix = "village_"
def categorys = ["190108","190109"]//村庄
def prefix = "company_"
def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
// def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
def totalCount = new AtomicLong()
static main(args) {
@@ -118,7 +118,8 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys =["无锡"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"]
def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"]
def rets = []
rets << headers
for(def s : districts){
def arrs = s.split("_")
def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2])
def ret = []
ret << s
if(m){
ret << m.city
ret << m.district
ret << m.name
if(m.alias){
ret << m.alias.toString().replace("[","").replace("]","")
}else{
ret << ""
}
ret << m.address
if(m.addressAlias){
ret << m.addressAlias.toString().replace("[","").replace("]","")
}else{
ret << ""
}
ret << m.avgPrice
ret << m.lng
ret << m.lat
ret << m.block
ret << m.loopLine
ret << m.propertyType
ret << m.buildingType
ret << m.commercialArea
ret << m.totalUnitCount
ret << m.managementFee
ret << m.parking
ret << m.parkingDescription
ret << m.ownershipPeriod
ret << m.ownershipType
ret << m.startBuildTime
ret << m.startSellingTime
ret << m.finishBuildTime
ret << m.moveinTime
ret << m.floorSpace
ret << m.constructionSpace
ret << m.plotRatio
ret << m.greenPercentage
ret << m.preSalePermit
ret << m.developer
ret << m.propertyManagement
ret << m.decorationStandard
ret << m.buildingStructure
ret << m.supportingFacilities
ret << m.transportation
ret << m.relatedInfo
ret << m.description
ret << m.salesOfficeAdd
ret << m.salesPhoneNumber
ret << m.discount
ret << m.buildingCount
ret << m.finishYear
}
rets << ret
}
ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据")