@@ -258,8 +258,9 @@ public class BaiDuApiUtils {
|
||||
params.put("coords", lng.toString() + "," + lat.toString());
|
||||
params.put("from", String.valueOf(from));
|
||||
HttpRequest request = new HttpRequest();
|
||||
String result = null;
|
||||
try{
|
||||
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(result);
|
||||
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
|
||||
changeAK();
|
||||
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 小区建筑类型数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommbuildtypeBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"])
|
||||
def rows = createDatas(cityCodeDatas, distCodeMap, dists)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
}
|
||||
|
||||
def createDatas(cityCodeDatas, distCodeMap, dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDatas, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
|
||||
println dist.city +" "+dist.region+" "+dist.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(cityCodeDatas, distCodeMap, dist){
|
||||
def ret = []
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << dist._id.toString()
|
||||
ret << CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 小区数据同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists)
|
||||
rows.addAll(rets)
|
||||
println city +" "+rets.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
allDatas << headers
|
||||
allDatas.addAll(rows)
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDatas, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(def cityCodeDatas, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
try{
|
||||
ret << cityCodeDatas.city_code
|
||||
if(dist.region){
|
||||
ret << distCodeMap.get(dist.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist._id.toString()
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "物业小区"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
}catch(Exception e){
|
||||
println dist.city + " "+dist.name
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 成交数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataDealBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
for(def city : citys){
|
||||
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
// def trades = getCityData(city)
|
||||
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
|
||||
def youDataTrade = getYouDaData()
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
|
||||
println city +" "+youdaDatas.size()
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rets)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(rows)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(source == "youda"){
|
||||
result << converYouDaData(trade, cityCodeDatas, distCodeMap)
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
}else{
|
||||
result << converData(trade, cityCodeDatas, distCodeMap)
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def obj = CommonDatautils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << trade._id.toString()
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(trade.fjRegion){
|
||||
ret << distCodeMap.get(trade.fjRegion)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.region
|
||||
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << trade._id.toString()
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(trade.region){
|
||||
ret << distCodeMap.get(trade.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.region
|
||||
if(trade.address){
|
||||
if(trade.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.dist_id.toString()
|
||||
ret << trade.master_name
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
|
||||
ret << ""//期数
|
||||
ret << trade.floor
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.avg_price
|
||||
ret << trade.total_price
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
def root = sourceMap.get(trade.source)
|
||||
if(root){
|
||||
if(CcbCommonUtils.rootMap.get(root)){
|
||||
ret << CcbCommonUtils.rootMap.get(root)
|
||||
}else{
|
||||
ret << "99"
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.total_price
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getHouseRoom(def room, def hall, def toilet){
|
||||
def s = ""
|
||||
if(room){
|
||||
s = s + room +"室"
|
||||
}
|
||||
if(hall){
|
||||
s = s + hall +"厅"
|
||||
}
|
||||
if(toilet){
|
||||
s = s + toilet +"卫"
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
def getCityData(def city){
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"])
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date()))
|
||||
if(trades){
|
||||
tradesList.addAll(trades)
|
||||
}
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return tradesList
|
||||
}
|
||||
|
||||
def getYouDaData(){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate));
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
for(def obj : dbcursor){
|
||||
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
|
||||
rets << obj;
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 区域数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDistrictBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataDistrictBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def datas = findDistrictCodeValue(city)
|
||||
if(datas){
|
||||
rows.addAll(datas)
|
||||
}
|
||||
println city+" "+ datas.size()
|
||||
}
|
||||
|
||||
def allDatas = createDatas(rows)
|
||||
|
||||
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8",false);
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
def allRows = []
|
||||
def index = 1
|
||||
rows.each { m ->
|
||||
def data = []
|
||||
data << m.dist_code
|
||||
data << m.dist_name
|
||||
data << m.dist_spell
|
||||
data << m.city_id
|
||||
data << m.dist_code
|
||||
data << index
|
||||
data << ""
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "FJW"
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
index++
|
||||
allRows << data
|
||||
}
|
||||
return allRows;
|
||||
}
|
||||
|
||||
def findDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
q.put("district", new BasicDBObject('$ne',"市辖区"))
|
||||
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def lists = []
|
||||
for(def db : dbcursor){
|
||||
if(db.fangjia_district && db.fangjia_district=="二道"){
|
||||
println db.fangjia_district+" "+city
|
||||
}
|
||||
if(db.district_code){
|
||||
def m = [:]
|
||||
m.id = db._id.toString()
|
||||
m.dist_name = db.fangjia_district
|
||||
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
|
||||
m.city_id = db.city_code
|
||||
m.dist_code = db.district_code
|
||||
lists << m
|
||||
}
|
||||
}
|
||||
return lists;
|
||||
}
|
||||
}
|
||||
+521
@@ -0,0 +1,521 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataForsaleBatch extends BaseBatch{
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731);
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeDatas.city_code
|
||||
def regionCode = distCodeMap.get(rMap.region)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = rMap.region
|
||||
m.region_code = regionCode
|
||||
m.district_id = rMap.get("_id").toString()
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
}
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
println city + " 房源总量:"+retMap.size()
|
||||
// def allDatas = createSellDatas(city, retMap.values());
|
||||
def allDatas = retMap.values()
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true);
|
||||
println "txt文件生成 "
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createSellDatas(def city, def values){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(values.size())
|
||||
println city +"总房源量:"+values.size()
|
||||
|
||||
def datas = Collections.synchronizedList(new ArrayList());
|
||||
for(def i=0;i<values.size();i++){
|
||||
final def m = values[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
datas << converData(m)
|
||||
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return datas ;
|
||||
}
|
||||
|
||||
def converData(def m){
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
data << m.region
|
||||
data << m.district_id
|
||||
data << m.district_name
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.address==null?"":m.address==null
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << ""
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
data << m.hall==null?"":m.hall
|
||||
data << ""
|
||||
data << m.toilet==null?"":m.toilet
|
||||
data << m.direction_code==null?"":m.direction_code
|
||||
data << m.fitment_code==null?"":m.fitment_code
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.p_date==null?"":m.p_date
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.root
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.c_date==null?"":m.c_date
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
data << ""
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
if(seawater.height && seawater.height<1){
|
||||
continue;
|
||||
}
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=5 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
|
||||
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String csvHandlerStr(String str) {
|
||||
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
String tempDescription=str;
|
||||
//如果有逗号
|
||||
if(str.contains(",")||str.contains(",")){
|
||||
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if(str.contains("\"")){
|
||||
// tempDescription=str.replace("\"", "\"\"");
|
||||
tempDescription=str.replace("\"", "");
|
||||
}
|
||||
//在将逗号转义
|
||||
tempDescription="\""+tempDescription+"\"";
|
||||
}
|
||||
return tempDescription;
|
||||
}
|
||||
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
}
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
|
||||
// public static def citys = ["武汉"]
|
||||
|
||||
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
|
||||
|
||||
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
public static def findCityCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def count = dbcursor.count()
|
||||
if(count >1){
|
||||
def data = null
|
||||
for(def db : dbcursor){
|
||||
if(db.city_code){
|
||||
data = db
|
||||
break
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
public static def findAllDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
q.put("district", new BasicDBObject('$ne',"市辖区"))
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
map.put(db.fangjia_district, db.district_code)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public static def findPersonPrice(city, date){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)));
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
public static def findTradesDetail(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
d.build_type = dist.build_type
|
||||
d.build_structure = dist.build_structure
|
||||
d.house_roperty = dist.house_roperty
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 城市数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class CityBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new CityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def index = 1
|
||||
for(def city : citys){
|
||||
def data = CcbCommonUtils.findCityCodeValue(city)
|
||||
def map = [:]
|
||||
map.name = city
|
||||
if(data){
|
||||
map.city_code = data.city_code
|
||||
map.prov_code = data.province_code
|
||||
}
|
||||
def cityObj = findCityInfo(city)
|
||||
if(cityObj){
|
||||
map.city_spell = cityObj.spell
|
||||
map.city_level = cityObj.city_grade
|
||||
map.seq = index
|
||||
map.ifcapital = isCapital(cityObj.province, city)
|
||||
map.ifmunicipal = ifMunicipal(city)
|
||||
}
|
||||
index++
|
||||
rows << map
|
||||
}
|
||||
|
||||
def allDatas = createDatas(rows)
|
||||
def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
def allRows = []
|
||||
rows.each { m ->
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.name
|
||||
data << m.city_spell
|
||||
data << m.prov_code
|
||||
data << m.city_code
|
||||
data << m.city_level
|
||||
data << m.seq
|
||||
data << CcbCodeUtils.parseFlag(m.ifcapital)
|
||||
data << CcbCodeUtils.parseFlag(m.ifmunicipal)
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "fjw"
|
||||
allRows << data
|
||||
}
|
||||
return allRows;
|
||||
}
|
||||
|
||||
def findCityInfo(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("display", city)
|
||||
return CcbCommonUtils.SERVER_AREA_COLL.findOne(q)
|
||||
}
|
||||
|
||||
def isCapital(def prov, def city){
|
||||
def provCity = CcbCommonUtils.provCityMap.get(prov)
|
||||
if(provCity && provCity==city){
|
||||
return "是"
|
||||
}
|
||||
return "否"
|
||||
}
|
||||
|
||||
def ifMunicipal(def city){
|
||||
if(["上海", "北京", "天津", "重庆"].contains(city)){
|
||||
return "是"
|
||||
}
|
||||
return "否"
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 估价系数数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class EvaluateRateBatch extends BaseBatch{
|
||||
|
||||
static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new EvaluateRateBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def allDatas = createDatas(findEvaluationRate())
|
||||
def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def createDatas(def datas){
|
||||
def allRows = []
|
||||
datas.each { m ->
|
||||
def data = []
|
||||
data << m.id.toString()
|
||||
data << m.rate_type
|
||||
data << m.rate
|
||||
data << m.rank
|
||||
data << "全国"
|
||||
data << m.rank_remark
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "FJW"
|
||||
allRows << data
|
||||
}
|
||||
return allRows
|
||||
}
|
||||
|
||||
private def findEvaluationRate(){
|
||||
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'";
|
||||
def rets = []
|
||||
sqlInstance.eachRow(sql,{
|
||||
rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated]
|
||||
});
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 人工指导价同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GuiDingPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false);
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(rows)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(dist){
|
||||
def ret = []
|
||||
ret << ps._id.toString()
|
||||
ret << dist._id.toString()
|
||||
ret << dist.name
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(dist.region){
|
||||
ret << distCodeMap.get(dist.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << dist.region
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.guiding_value
|
||||
ret << ps.guiding_person
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 分期数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class StagingDistrictBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new StagingDistrictBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"])
|
||||
def fenQiMap = getFenQiMap(dists);
|
||||
def rets = createCityData(dists, fenQiMap)
|
||||
rows.addAll(rets)
|
||||
println city +" 分期量:"+rets.size()
|
||||
}
|
||||
|
||||
|
||||
def results =[]
|
||||
def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
|
||||
results << headers
|
||||
results.addAll(rows)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8");
|
||||
|
||||
}
|
||||
|
||||
def createCityData(def dists, def fenQiMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
if(d.master_district_name && d.master_district_name!="master"){
|
||||
def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name)
|
||||
if(masterDist){
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << d._id.toString()
|
||||
ret << masterDist._id.toString()
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
// else{
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("master_district_name", null)
|
||||
// udata.put("m_date", new Date())
|
||||
// println d.city +" "+d.name+" "+d._id.toString()
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getFenQiMap(def dists){
|
||||
def fenQiMap = [:]
|
||||
for(def d : dists){
|
||||
if(d.master_district_name){
|
||||
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
}
|
||||
return fenQiMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
import groovy.sql.Sql
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
for(def city : citys) {
|
||||
def data = CcbCommonUtils.findCityCodeValue(city)
|
||||
def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
|
||||
|
||||
def count = 0
|
||||
sqlInstance.execute(sql,{
|
||||
count = it.count
|
||||
});
|
||||
println city +" "+count
|
||||
}
|
||||
|
||||
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 第三方均价数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThirdPartyPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index++
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"])
|
||||
def thirdPriceMap = findThirdPartyPrice(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
|
||||
rows.addAll(rets)
|
||||
println index+" "+city+" "+rets.size()
|
||||
}
|
||||
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def ret = []
|
||||
ret << third._id.toString()
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(d.region){
|
||||
ret << distCodeMap.get(d.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << d.region
|
||||
ret << third.s_date
|
||||
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
ret << third.zhugezhaofang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
ret << third.anjuke_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
ret << third.soufang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.lianjia_esf_price){
|
||||
ret << third.lianjia_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.cityhouse_esf_price){
|
||||
ret << third.cityhouse_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.maitian_esf_price){
|
||||
ret << third.maitian_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.beike_esf_price){
|
||||
ret << third.beike_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.get("365house_esf_price")){
|
||||
ret << third.get("365house_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.zhongyuan_esf_price){
|
||||
ret << third.zhongyuan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.tuboshi_esf_price){
|
||||
ret << third.tuboshi_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
ret << third.qfang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangtan_esf_price){
|
||||
ret << third.fangtan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangduoduo_esf_price){
|
||||
ret << third.fangduoduo_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
ret << third.get("5i5j_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", curDate)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
map.put(db.region+"##"+db.name, db)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+31
-28
@@ -74,20 +74,20 @@ class AddThirdPriceDistrict {
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceDistrict();
|
||||
// task.run()
|
||||
task.run()
|
||||
|
||||
def mapName = "锦华星光苑3,4幢"
|
||||
// def mapName = "锦华星光苑3,4幢"
|
||||
//// println task.pattern("港新花苑A1区");
|
||||
println mapName.replaceAll("\\w+-?\\w+(幢)", "")
|
||||
println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
|
||||
// println mapName.replaceAll("\\w+-?\\w+(幢)", "")
|
||||
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
|
||||
}
|
||||
def run(){
|
||||
// exportDatas();
|
||||
exportDatas();
|
||||
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
// addInfo();
|
||||
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
|
||||
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
|
||||
addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
}
|
||||
|
||||
def addCrawlMapInfo( def path){
|
||||
@@ -135,29 +135,32 @@ class AddThirdPriceDistrict {
|
||||
|
||||
|
||||
def exportDatas(){
|
||||
def city = "无锡"
|
||||
def datas = findMatchErrorDatas(city);
|
||||
println "共【"+datas.size()+"】条"
|
||||
datas = filterName(datas);
|
||||
println "过滤后共【"+datas.size()+"】条"
|
||||
def resMap = groupByName(datas);
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
|
||||
resMap.each {k,v->
|
||||
def ret = []
|
||||
def arrs = k.split("##")
|
||||
ret << arrs[0]
|
||||
ret << arrs[1]
|
||||
if(arrs.size()==3 && arrs[2]){
|
||||
ret << arrs[2]
|
||||
}else{
|
||||
ret << ""
|
||||
//跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞",
|
||||
def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
for(def city : citys){
|
||||
def datas = findMatchErrorDatas(city);
|
||||
println "共【"+datas.size()+"】条"
|
||||
datas = filterName(datas);
|
||||
println "过滤后共【"+datas.size()+"】条"
|
||||
def resMap = groupByName(datas);
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
|
||||
resMap.each {k,v->
|
||||
def ret = []
|
||||
def arrs = k.split("##")
|
||||
ret << arrs[0]
|
||||
ret << arrs[1]
|
||||
if(arrs.size()==3 && arrs[2]){
|
||||
ret << arrs[2]
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << resAddrMap.get(k)
|
||||
ret << v.join(",")
|
||||
rets << ret
|
||||
}
|
||||
ret << resAddrMap.get(k)
|
||||
ret << v.join(",")
|
||||
rets << ret
|
||||
ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后")
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后")
|
||||
}
|
||||
|
||||
def filterName(datas){
|
||||
@@ -465,7 +468,7 @@ class AddThirdPriceDistrict {
|
||||
def findMatchErrorDatas(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date",202005)
|
||||
q.put("s_date",new BasicDBObject('$gte', 202101))
|
||||
// q.put("name","博济聚创科技园")
|
||||
// q.put("source","贝壳找房")
|
||||
def dbcursor = matchErrorLogColl.find(q)
|
||||
|
||||
@@ -7,7 +7,7 @@ import java.sql.ResultSet
|
||||
class EsSearchUtils {
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -394,6 +394,7 @@ class FiledStrand {
|
||||
}
|
||||
return estateCharege;
|
||||
}
|
||||
|
||||
static def processHouseholdCount(def household_count, def estateType2){
|
||||
if(household_count){
|
||||
household_count = household_count.toInteger()
|
||||
|
||||
@@ -56,11 +56,11 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
// def prefix = "road_"
|
||||
// def categorys = ["190301"]//道路相关
|
||||
|
||||
// def prefix = "village_"
|
||||
// def categorys = ["190108","190109"]//村庄
|
||||
def prefix = "village_"
|
||||
def categorys = ["190108","190109"]//村庄
|
||||
|
||||
def prefix = "company_"
|
||||
def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
|
||||
// def prefix = "company_"
|
||||
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
@@ -118,7 +118,8 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys =["无锡"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
|
||||
@@ -29,6 +29,8 @@ class OpenApiUtils {
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.id = dto.id
|
||||
rMap.lng = dto.lng
|
||||
rMap.lat = dto.lat
|
||||
if(dto.address && dto.address!="null"){
|
||||
rMap.address = dto.address
|
||||
}
|
||||
|
||||
+36
-1
@@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
class WuXiAddressTest {
|
||||
static main(args) {
|
||||
def task = new WuXiAddressTest()
|
||||
task.run()
|
||||
//task.run()
|
||||
task.addAddress()
|
||||
}
|
||||
|
||||
def addAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d->
|
||||
def prov = d['C']
|
||||
def city = d['D']
|
||||
def region = d['E']
|
||||
def address = d['B']
|
||||
def road = d['J']
|
||||
def street = d['R']
|
||||
|
||||
def newAddress = prov+"省"+city +"市"
|
||||
if(region == "惠山"){
|
||||
newAddress = newAddress + region +"区"
|
||||
}else{
|
||||
newAddress = newAddress + region +"市"
|
||||
}
|
||||
|
||||
d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "")
|
||||
def dist = d['U']
|
||||
|
||||
def s1 = null
|
||||
if(road){
|
||||
s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","")
|
||||
newAddress = newAddress + street + dist + road + s1
|
||||
}else if(d['L']){
|
||||
s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","")
|
||||
newAddress = newAddress + street + dist + d['L'] + s1
|
||||
}
|
||||
|
||||
d['V'] = newAddress
|
||||
},0)
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -18,6 +51,8 @@ class WuXiAddressTest {
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
|
||||
if(!d['S']){
|
||||
def locationStr = getLocationInfo(d['D'], d['Q'])
|
||||
|
||||
+3
-3
@@ -47,11 +47,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-05-01"
|
||||
static def statDate = "2021-06-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
@@ -7,6 +7,13 @@ import java.util.regex.Pattern
|
||||
class CcbCodeUtils {
|
||||
|
||||
|
||||
//否:0 是:1
|
||||
static def parseFlag(String flag){
|
||||
if(flag && flag == "是"){
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static def parseBuildYear(String buildDate) {
|
||||
if(buildDate){
|
||||
@@ -53,6 +60,44 @@ class CcbCodeUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
// 高层塔楼 0101 增加“超高层” 0108
|
||||
// 高层板楼 0102 增加“老洋房” 0109
|
||||
// 小高层板楼 0103 增加“花园洋房” 0110
|
||||
// 多层板楼 0104 增加“四合院” 0111
|
||||
// 独栋别墅 0105 增加“老式别墅洋房” 0112
|
||||
// 联排别墅 0106
|
||||
// 叠拼别墅 0107
|
||||
// 其他建筑形式 0199
|
||||
|
||||
static def parseBuildType(def buildType){
|
||||
if(buildType){
|
||||
if(buildType.contains("老式别墅洋房")){
|
||||
return "0112"
|
||||
}else if(buildType.contains("四合院")){
|
||||
return "0111"
|
||||
}else if(buildType.contains("花园洋房")){
|
||||
return "0110"
|
||||
}else if(buildType.contains("老洋房")){
|
||||
return "0109"
|
||||
}else if(buildType.contains("叠拼")){
|
||||
return "0107"
|
||||
}else if(buildType.contains("联排")){
|
||||
return "0106"
|
||||
}else if(buildType.contains("独栋")){
|
||||
return "0105"
|
||||
}else if(buildType.contains("多层")){
|
||||
return "0104"
|
||||
}else if(buildType.contains("超高层")){
|
||||
return "0108"
|
||||
}else if(buildType.contains("小高层")){
|
||||
return "0103"
|
||||
}else if(buildType.contains("高层")){
|
||||
return "0102"
|
||||
}
|
||||
}
|
||||
return "0199"
|
||||
}
|
||||
|
||||
|
||||
//朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399
|
||||
static def getDirectionCode(def direction){
|
||||
@@ -161,4 +206,19 @@ class CcbCodeUtils {
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//房屋用途
|
||||
static def parseHouseUse(String houseUse){
|
||||
if(houseUse){
|
||||
if(houseUse.contains("住宅")){
|
||||
return 11;
|
||||
}else if(houseUse.contains("商住")){
|
||||
return 13;
|
||||
}else{
|
||||
return 99
|
||||
}
|
||||
}
|
||||
return 99
|
||||
}
|
||||
}
|
||||
|
||||
+15
-15
@@ -3,13 +3,13 @@ package com.fangjia.script.gj.ccb
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FileUtils {
|
||||
static def batchCreateFile(ret, fileName, headers,charset){
|
||||
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
}
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
sb.append(headers.join("|@|"))
|
||||
@@ -20,18 +20,18 @@ class FileUtils {
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
println index
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
class BestDistrictPriceExport {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new BestDistrictPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "佛山"
|
||||
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
def rets = []
|
||||
rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"]
|
||||
def distMap = findCreditHightDistrict2(city, "20210711");
|
||||
distMap.each{region, dists->
|
||||
|
||||
if(region=="三水"){
|
||||
def addIndex = 0
|
||||
for(def dist : dists){
|
||||
if(addIndex <=30){
|
||||
def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name)
|
||||
def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711")
|
||||
if(otherPrice){
|
||||
def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price
|
||||
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
|
||||
def ret = []
|
||||
ret << "佛山"
|
||||
ret << dist.region
|
||||
ret << dist.district_name
|
||||
ret << dist.estate_type
|
||||
for(def weekend : months){
|
||||
def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend)
|
||||
if(obj){
|
||||
ret << obj.avg_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
ret << otherPrice.lianjia_esf_price
|
||||
ret << stat.credit
|
||||
rets << ret
|
||||
addIndex++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
|
||||
def findDistrictPrice(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def obj = DIST_SELL_STAT.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def findCreditHightDistrict(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def typeMap = retMaps.get(d.region)
|
||||
if(!typeMap){
|
||||
typeMap = [:]
|
||||
retMaps.put(d.region, typeMap)
|
||||
}
|
||||
|
||||
def rets = typeMap.get(d.estate_type)
|
||||
if(!rets){
|
||||
rets = []
|
||||
typeMap.put(d.estate_type, rets)
|
||||
}
|
||||
|
||||
def count = 0
|
||||
typeMap.each {k,v->
|
||||
if(v){
|
||||
count = count + v.size()
|
||||
}
|
||||
}
|
||||
|
||||
if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}else if(rets.size() <= 20){
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}
|
||||
|
||||
def total = 0
|
||||
retMaps.each {k,m->
|
||||
m.each {t,v->
|
||||
total = total + v.size()
|
||||
}
|
||||
}
|
||||
if(total>100){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
def rows = []
|
||||
retMaps.each {k,m->
|
||||
m.each {t,v->
|
||||
rows.addAll(v)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
def findCreditHightDistrict2(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.estate_type = d.estate_type
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}
|
||||
//
|
||||
// def rows = []
|
||||
// retMaps.each {k,v->
|
||||
// rows.addAll(v)
|
||||
// }
|
||||
return retMaps;
|
||||
}
|
||||
|
||||
def findNearThirdPartyPrice(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", 202106)
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
return AVERAGE_STATISTICS.findOne(q)
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
|
||||
def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"]
|
||||
|
||||
def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
|
||||
for(def s : districts){
|
||||
def arrs = s.split("_")
|
||||
def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2])
|
||||
def ret = []
|
||||
ret << s
|
||||
if(m){
|
||||
ret << m.city
|
||||
ret << m.district
|
||||
ret << m.name
|
||||
if(m.alias){
|
||||
ret << m.alias.toString().replace("[","").replace("]","")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << m.address
|
||||
if(m.addressAlias){
|
||||
ret << m.addressAlias.toString().replace("[","").replace("]","")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << m.avgPrice
|
||||
ret << m.lng
|
||||
ret << m.lat
|
||||
ret << m.block
|
||||
ret << m.loopLine
|
||||
ret << m.propertyType
|
||||
ret << m.buildingType
|
||||
ret << m.commercialArea
|
||||
ret << m.totalUnitCount
|
||||
ret << m.managementFee
|
||||
ret << m.parking
|
||||
ret << m.parkingDescription
|
||||
ret << m.ownershipPeriod
|
||||
ret << m.ownershipType
|
||||
ret << m.startBuildTime
|
||||
ret << m.startSellingTime
|
||||
ret << m.finishBuildTime
|
||||
ret << m.moveinTime
|
||||
ret << m.floorSpace
|
||||
ret << m.constructionSpace
|
||||
ret << m.plotRatio
|
||||
ret << m.greenPercentage
|
||||
ret << m.preSalePermit
|
||||
ret << m.developer
|
||||
ret << m.propertyManagement
|
||||
ret << m.decorationStandard
|
||||
ret << m.buildingStructure
|
||||
ret << m.supportingFacilities
|
||||
ret << m.transportation
|
||||
ret << m.relatedInfo
|
||||
ret << m.description
|
||||
ret << m.salesOfficeAdd
|
||||
ret << m.salesPhoneNumber
|
||||
ret << m.discount
|
||||
ret << m.buildingCount
|
||||
ret << m.finishYear
|
||||
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据")
|
||||
|
||||
|
||||
|
||||
|
||||
+18
-6
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -10,22 +11,23 @@ import com.mongodb.DBCollection
|
||||
class RegionPriceExport {
|
||||
|
||||
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
|
||||
DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new RegionPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def months = [202010,202011,202012,202101,202102,202103]
|
||||
def regions = CommonUtils.getAllRegions("福州","region")
|
||||
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
def regions = CommonUtils.getAllRegions("佛山","region")
|
||||
def rets = []
|
||||
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
|
||||
rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
regions.each{region,values->
|
||||
def data = []
|
||||
data << "福州"
|
||||
data << "佛山"
|
||||
data << region
|
||||
for(def month :months){
|
||||
def dbo = findPrice("福州", region, month);
|
||||
def dbo = findWeekendPrice("佛山", region, month);
|
||||
if(dbo){
|
||||
data << dbo.avg_price
|
||||
}else{
|
||||
@@ -34,7 +36,7 @@ class RegionPriceExport {
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
|
||||
ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
def findPrice(city, region, month){
|
||||
@@ -45,4 +47,14 @@ class RegionPriceExport {
|
||||
def dbo = REGION_SELL_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def findWeekendPrice(city, region, month){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("cat", "区域")
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
|
||||
def dbo = REGION_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
}
|
||||
|
||||
+25
-21
@@ -1,6 +1,8 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -13,7 +15,7 @@ class ExportData {
|
||||
|
||||
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -24,8 +26,8 @@ class ExportData {
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportData();
|
||||
task.exportThird()
|
||||
// task.exportMap();
|
||||
// task.exportThird()
|
||||
task.exportMap();
|
||||
}
|
||||
|
||||
def exportMap(){
|
||||
@@ -51,19 +53,18 @@ class ExportData {
|
||||
//"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特"
|
||||
//宁夏 ----- 中卫 固原 石嘴山 吴忠 银川
|
||||
//青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁
|
||||
def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
|
||||
"中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
|
||||
"防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
|
||||
//"沈阳","大连","抚顺","鞍山"
|
||||
//"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆"
|
||||
//"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛",
|
||||
//"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州",
|
||||
//"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城",
|
||||
// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
|
||||
// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
|
||||
// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
|
||||
def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"];
|
||||
|
||||
for(def city : citys){
|
||||
def path = "D:\\数据\\小区\\地图\\${city}.xlsx";
|
||||
def datas = findAllDatas(city, null)
|
||||
println "${city}共计:"+datas.size()
|
||||
ExcelUtils.write(path, datas, "数据")
|
||||
}
|
||||
|
||||
def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx";
|
||||
def datas = findAllDatas(citys, null)
|
||||
println "共计:"+datas.size()
|
||||
ExcelUtils.write(path, datas, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -98,9 +99,9 @@ class ExportData {
|
||||
|
||||
|
||||
|
||||
def findAllDatas(def citys, def pname){
|
||||
def findAllDatas(def city, def pname){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'")
|
||||
// sqlBuffer.append(" and pname = '"+pname+"'")
|
||||
// sqlBuffer.append(" and cityname = '"+pname+"'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
@@ -113,16 +114,19 @@ class ExportData {
|
||||
if(source && source=="高德"){
|
||||
println index
|
||||
def type = rs.getObject("type")
|
||||
def city = rs.getString("cityname")
|
||||
if(citys.contains(city) && type.contains("商务住宅")){
|
||||
//def city = rs.getString("cityname")
|
||||
if(type.contains("住宅")){
|
||||
def ret = []
|
||||
ret << city
|
||||
ret << rs.getString("adname")
|
||||
ret << rs.getString("name")
|
||||
ret << rs.getString("address")
|
||||
ret << rs.getString("type")
|
||||
ret << rs.getFloat("location.lon")
|
||||
ret << rs.getFloat("location.lat")
|
||||
def loaction = rs.getObject("location")
|
||||
if(loaction){
|
||||
ret << DataHandleUtils.getDoubleRound(6, loaction.x)
|
||||
ret << DataHandleUtils.getDoubleRound(6, loaction.y)
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -68,13 +68,16 @@ class FangjiaClientUtils {
|
||||
|
||||
public static def addressStand(def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
|
||||
+18
-13
@@ -1,9 +1,12 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
|
||||
|
||||
import java.util.regex.Pattern
|
||||
@@ -20,20 +23,13 @@ class MapDistrictProcess {
|
||||
* @return
|
||||
*/
|
||||
def run(){
|
||||
//"江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx","安徽地图.xlsx",
|
||||
//"上海地图.xlsx","北京地图.xlsx"
|
||||
//"安徽地图.xlsx","江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx",
|
||||
|
||||
//"重庆地图.xlsx","天津地图.xlsx",
|
||||
//"上海地图.xlsx","北京地图.xlsx",
|
||||
//"河北地图.xlsx","河南地图.xlsx","湖北地图.xlsx","湖南地图.xlsx","广西地图.xlsx",
|
||||
//"吉林地图.xlsx","辽宁地图.xlsx""辽宁地图.xlsx","吉林地图.xlsx","黑龙江地图.xlsx","山西地图.xlsx","海南地图.xlsx"]
|
||||
def files = ["青海地图.xlsx","宁夏地图.xlsx","内蒙古地图.xlsx","云南地图.xlsx","陕西地图.xlsx","新疆地图.xlsx","四川地图.xlsx"]
|
||||
def files = ["武汉.xlsx"]
|
||||
for(def fileName : files){
|
||||
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
|
||||
typeFilter(filePath)
|
||||
nameStandard(filePath);
|
||||
dataFilter(filePath)
|
||||
// typeFilter(filePath)
|
||||
// nameStandard(filePath);
|
||||
// dataFilter(filePath)
|
||||
matchDistDatas(filePath)
|
||||
houseCalculate(filePath);
|
||||
////// //统计是否存在多个区域
|
||||
@@ -114,15 +110,24 @@ class MapDistrictProcess {
|
||||
|
||||
def matchDistDatas(def path){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(path,{ List row ->
|
||||
ExcelUtils.modify(path,{ List row ->
|
||||
index ++
|
||||
if(!row['I'] && !row['J']){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],);
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
|
||||
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
|
||||
row['J'] = rMap.city
|
||||
row['K'] = rMap.region
|
||||
row['L'] = rMap.name
|
||||
row['M'] = rMap.address
|
||||
|
||||
if(rMap.lat){
|
||||
def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
|
||||
def maplng = baidu.x
|
||||
def maplat = baidu.y
|
||||
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
|
||||
row['Y'] = score
|
||||
}
|
||||
}
|
||||
println index +" "+row
|
||||
}
|
||||
|
||||
+8
-5
@@ -26,15 +26,18 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && address){
|
||||
// if (findPattern(address)){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
// }
|
||||
}
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
||||
}
|
||||
|
||||
if(!rMap && address){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
if(rMap && rMap.region && !region.contains(rMap.region)){
|
||||
rMap = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class SourceDistDataMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new SourceDistDataMatch();
|
||||
// task.run()
|
||||
task.mapRun()
|
||||
// task.filterName()
|
||||
// task.filterType()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(obj){
|
||||
d['R'] = 1
|
||||
}else{
|
||||
def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709)
|
||||
if(distCount){
|
||||
d['W'] = distCount
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def mapRun(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N'] && !d['F']){
|
||||
def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']]
|
||||
def locationMap = matchMap(dist)
|
||||
if(locationMap){
|
||||
d['F'] = locationMap.source
|
||||
d['G'] = locationMap.region
|
||||
d['H'] = locationMap.mapName
|
||||
d['I'] = locationMap.address
|
||||
d['J'] = locationMap.type
|
||||
d['K'] = locationMap.lng
|
||||
d['L'] = locationMap.lat
|
||||
d['M'] = locationMap.score
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def filterName(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N']) {
|
||||
def name = d['C']
|
||||
if (name) {
|
||||
def flag = NameProcess.filterName(name)
|
||||
if (!flag) {
|
||||
d['N'] = 4
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def filterType(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N']){
|
||||
def type = d['J']
|
||||
if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){
|
||||
|
||||
}else if(type && type.contains("商务住宅") && !type.contains("商住两用")){
|
||||
def count = 0
|
||||
if(d['S'] && d['S']!='null'){
|
||||
count = d['S'].toDouble().toInteger()
|
||||
}
|
||||
if(count<10){
|
||||
d['N'] = 3
|
||||
}
|
||||
}else{
|
||||
if(type){
|
||||
d['N'] = 3
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def matchMap(def dist){
|
||||
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
|
||||
if(!gaodeRest && dist.address){
|
||||
gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德")
|
||||
}
|
||||
def gdlocationMap = null
|
||||
def bdlocationMap = null
|
||||
if(gaodeRest){
|
||||
gdlocationMap = getMapLocation2(dist, gaodeRest);
|
||||
if(gdlocationMap){
|
||||
gdlocationMap.source = "高德"
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
if(baiduRest){
|
||||
bdlocationMap = getMapLocation2(dist, baiduRest);
|
||||
if(bdlocationMap){
|
||||
bdlocationMap.source = "百度"
|
||||
}
|
||||
}
|
||||
|
||||
if(gdlocationMap && bdlocationMap){
|
||||
if(gdlocationMap.score>=bdlocationMap.score){
|
||||
return gdlocationMap
|
||||
}else{
|
||||
return bdlocationMap
|
||||
}
|
||||
}else if(gdlocationMap){
|
||||
return gdlocationMap
|
||||
}else if(bdlocationMap){
|
||||
return bdlocationMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
def getMapLocation2(def dist, def rets){
|
||||
def data1 = findSimilarData(dist, rets, true)
|
||||
def data2 = findSimilarData(dist, rets, false)
|
||||
println data1
|
||||
println data2
|
||||
if(data1 && data1.score>1){
|
||||
return data1
|
||||
}else if(data2 && data2.score>1){
|
||||
return data2
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findSimilarData(def dist, def rets, def regionImpr){
|
||||
def lat = null
|
||||
def lng = null
|
||||
def region = null
|
||||
def mapName = null
|
||||
def address = null
|
||||
def score = 0
|
||||
def type = null
|
||||
def maxScore = 0
|
||||
def standardName = null
|
||||
for(def r : rets){
|
||||
if(!r.city){
|
||||
continue
|
||||
}
|
||||
if(!r.city.contains(dist.city)){
|
||||
continue
|
||||
}
|
||||
if(["公交车站"].contains(r.type)){
|
||||
continue
|
||||
}
|
||||
if(regionImpr && !r.district){
|
||||
continue
|
||||
}
|
||||
if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){
|
||||
continue
|
||||
}
|
||||
|
||||
def name = nameReplace(r.name, dist.city)
|
||||
|
||||
def distName = nameReplace(dist.name, dist.city)
|
||||
|
||||
def score3 = 0.0
|
||||
if(r.district && dist.region && r.district.contains(dist.region)){
|
||||
score3 = 0.2
|
||||
}
|
||||
|
||||
def score1 = 0.0;
|
||||
def score2 = 0.0
|
||||
if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){
|
||||
if(r.address){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2
|
||||
}
|
||||
}else{
|
||||
score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2;
|
||||
if(dist.address && r.address){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2
|
||||
}
|
||||
}
|
||||
|
||||
score3 = score1 + score2
|
||||
if(r.type && r.type.contains("住宅")){
|
||||
score3 = score3 + 0.2
|
||||
}
|
||||
if(score3 >maxScore){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
standardName = name
|
||||
address = r.address
|
||||
type = r.type
|
||||
score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10)
|
||||
maxScore = score3
|
||||
}
|
||||
println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3
|
||||
|
||||
}
|
||||
if(lng){
|
||||
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def nameReplace(def s, city){
|
||||
if(s.contains("(") && s.contains(")")){
|
||||
def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")"))
|
||||
if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){
|
||||
s = s.toString().substring(0, s.indexOf("("))
|
||||
}
|
||||
}
|
||||
def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim()
|
||||
if(name.startsWith(city)){
|
||||
def newName = name.replace(city+"市","").replace(city,"")
|
||||
if(newName && newName=="花园"){
|
||||
return name
|
||||
}else{
|
||||
name = newName
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -49,7 +49,7 @@ class ZsPersonProcess {
|
||||
// def loactionMap = task.matchMap(dist)
|
||||
// println loactionMap
|
||||
|
||||
// task.matchMap()
|
||||
task.matchMap()
|
||||
|
||||
// task.matchDb()
|
||||
// task.matchDb2()
|
||||
|
||||
+26
-28
@@ -19,9 +19,9 @@ class EsMonthHouseStat {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
// task.tradeRun()
|
||||
// task.houseRun()
|
||||
task.houseRun()
|
||||
|
||||
task.statHouseUpdateCount()
|
||||
// task.statHouseUpdateCount()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -99,37 +99,35 @@ class EsMonthHouseStat {
|
||||
}
|
||||
|
||||
def houseRun(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
|
||||
def city = d['B']
|
||||
if(city == "湖州"){
|
||||
def count = findHouseCount(city, 20210301, 20210601)
|
||||
d['E'] = count
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d->
|
||||
def city = d['A']
|
||||
def count = findHouseCount(city, 20190709, 20210709)
|
||||
d['E'] = count
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['D'] = precent+"%"
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['F'] = precent+"%"
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.fangjia.script.gj.region
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class MapAreaUpdate {
|
||||
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new MapAreaUpdate()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findAllCity()
|
||||
for(def city : rets){
|
||||
def retMap = findCityAndProvCode(city)
|
||||
if(retMap){
|
||||
update(retMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def update(retMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city",retMap.fangjia_city)
|
||||
q.put("district", "ÊÐÏ½Çø")
|
||||
def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city_code", retMap.city_code)
|
||||
udata.put("province_code", retMap.province_code)
|
||||
FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false)
|
||||
}
|
||||
}
|
||||
|
||||
def findCityAndProvCode(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city",city)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.city_code && d.province_code && d.district_code){
|
||||
map.put("fangjia_city",d.fangjia_city)
|
||||
map.put("city_code",d.city_code)
|
||||
map.put("province_code",d.province_code)
|
||||
break
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def findAllCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_district",null)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def citys = []
|
||||
for(def d : dbcursor){
|
||||
citys << d.fangjia_city
|
||||
}
|
||||
citys.unique()
|
||||
return citys
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""])
|
||||
citys.addAll(["¸£ÖÝ"])
|
||||
// getNoDataCitys();
|
||||
def cityurl = "http://www.zhuge.com/areachange/"
|
||||
def cityUrls = getCityUrlCrawal(cityurl);
|
||||
|
||||
@@ -140,6 +140,7 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) {
|
||||
|
||||
if (StringUtils.isBlank(input)) {
|
||||
|
||||
Reference in New Issue
Block a user