Merge branch 'script_gj' into 'master'

Script gj

See merge request !63
This commit is contained in:
2021-07-22 09:13:07 +08:00
35 changed files with 2779 additions and 130 deletions
@@ -258,8 +258,9 @@ public class BaiDuApiUtils {
params.put("coords", lng.toString() + "," + lat.toString());
params.put("from", String.valueOf(from));
HttpRequest request = new HttpRequest();
String result = null;
try{
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
changeAK();
@@ -0,0 +1,88 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 建行模型估值项目
* 小区建筑类型数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
runBatch(new BigdataCommbuildtypeBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rets = []
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"])
def rows = createDatas(cityCodeDatas, distCodeMap, dists)
rets.addAll(rows)
println city +" "+rows.size()
}
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false);
println "txt 生成"
}
def createDatas(cityCodeDatas, distCodeMap, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, distCodeMap, dist)
if(ret){
rets << ret
}
println dist.city +" "+dist.region+" "+dist.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(cityCodeDatas, distCodeMap, dist){
def ret = []
ret << cityCodeDatas.city_code
ret << dist._id.toString()
ret << CcbCodeUtils.parseBuildType(dist.build_type)
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
}
@@ -0,0 +1,164 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 小区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists)
rows.addAll(rets)
println city +" "+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def allDatas = []
allDatas << headers
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false);
println "txt 生成"
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, distCodeMap, dist)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(def cityCodeDatas, def distCodeMap, def dist){
def ret = []
try{
ret << cityCodeDatas.city_code
if(dist.region){
ret << distCodeMap.get(dist.region)
}else{
ret << ""
}
ret << dist._id.toString()
ret << dist.region
ret << dist.name
ret << dist.address
if(dist.lat){
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
}else{
ret << ""
ret << ""
}
ret << dist.all_area
ret << dist.build_area
ret << dist.virescence_percent
ret << dist.cubage_percent
ret << dist.car_site_count
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "物业小区"
ret << ""
if(dist.address){
if(dist.address.toString().endsWith("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << number
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
ret << ""
}
ret << 0
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << ""
ret << ""
ret << ""
}catch(Exception e){
println dist.city + " "+dist.name
e.printStackTrace()
}
return ret;
}
}
@@ -0,0 +1,416 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 成交数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static main(args) {
runBatch(new BigdataDealBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def trades = getCityData(city)
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other")
// rows.addAll(rets)
def youDataTrade = getYouDaData()
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rets)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false);
println "txt 生成"
def rets = []
rets << headers
rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
def result = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(source == "youda"){
result << converYouDaData(trade, cityCodeDatas, distCodeMap)
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}else{
result << converData(trade, cityCodeDatas, distCodeMap)
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def obj = CommonDatautils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.city_code
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
if(trade.fjRegion){
ret << distCodeMap.get(trade.fjRegion)
}else{
ret << ""
}
ret << trade.region
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().endsWith("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().endsWith("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss")
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
ret << cityCodeDatas.city_code
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
if(trade.region){
ret << distCodeMap.get(trade.region)
}else{
ret << ""
}
ret << trade.region
if(trade.address){
if(trade.address.toString().endsWith("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
ret << number
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
ret << ""
}
ret << trade.dist_id.toString()
ret << trade.master_name
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
ret << ""//期数
ret << trade.floor
ret << ""
ret << trade.height
ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.avg_price
ret << trade.total_price
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss")
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
def root = sourceMap.get(trade.source)
if(root){
if(CcbCommonUtils.rootMap.get(root)){
ret << CcbCommonUtils.rootMap.get(root)
}else{
ret << "99"
}
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.total_price
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def getHouseRoom(def room, def hall, def toilet){
def s = ""
if(room){
s = s + room +"室"
}
if(hall){
s = s + hall +"厅"
}
if(toilet){
s = s + toilet +"卫"
}
return s;
}
def getCityData(def city){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date()))
if(trades){
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return tradesList
}
def getYouDaData(){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate));
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
rets << obj;
}
}
return rets;
}
}
@@ -0,0 +1,95 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 建行模型估值项目
* 区域数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataDistrictBatch extends BaseBatch{
static main(args) {
runBatch(new BigdataDistrictBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def datas = findDistrictCodeValue(city)
if(datas){
rows.addAll(datas)
}
println city+" "+ datas.size()
}
def allDatas = createDatas(rows)
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8",false);
}
def createDatas(rows){
def allRows = []
def index = 1
rows.each { m ->
def data = []
data << m.dist_code
data << m.dist_name
data << m.dist_spell
data << m.city_id
data << m.dist_code
data << index
data << ""
data << ""
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
data << ""
data << ""
data << ""
data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
index++
allRows << data
}
return allRows;
}
def findDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
def lists = []
for(def db : dbcursor){
if(db.fangjia_district && db.fangjia_district=="二道"){
println db.fangjia_district+" "+city
}
if(db.district_code){
def m = [:]
m.id = db._id.toString()
m.dist_name = db.fangjia_district
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
m.city_id = db.city_code
m.dist_code = db.district_code
lists << m
}
}
return lists;
}
}
@@ -0,0 +1,521 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 出售房源数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataForsaleBatch extends BaseBatch{
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731);
}
}
def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret){
def cityCode = cityCodeDatas.city_code
def regionCode = distCodeMap.get(rMap.region)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = rMap.region
m.region_code = regionCode
m.district_id = rMap.get("_id").toString()
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
}
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
println city + " 房源总量:"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true);
println "txt文件生成 "
}
def createSellDatas(def city, def values){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(values.size())
println city +"总房源量:"+values.size()
def datas = Collections.synchronizedList(new ArrayList());
for(def i=0;i<values.size();i++){
final def m = values[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
datas << converData(m)
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return datas ;
}
def converData(def m){
def data = []
data << m.city_code
data << m.houseId
data << ""
data << m.city
data << m.region_code
data << m.region
data << m.district_id
data << m.district_name
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.address==null?"":m.address==null
data << m.build_year==null?"":m.build_year
data << m.proptype==null?"":m.proptype
data << m.propyears==null?"":m.propyears
data << m.avg_price==null?"":m.avg_price
data << m.total_price==null?"":m.total_price
data << ""
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.house_structure_code==null?"":m.house_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
data << m.house_type==null?"":m.house_type
data << m.room==null?"":m.room
data << m.hall==null?"":m.hall
data << ""
data << m.toilet==null?"":m.toilet
data << m.direction_code==null?"":m.direction_code
data << m.fitment_code==null?"":m.fitment_code
data << ""
data << ""
data << m.p_date==null?"":m.p_date
data << ""
data << ""
data << m.root
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.c_date==null?"":m.c_date
data << m.m_date==null?"":m.m_date
data << m.source
data << ""
return data
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
if(seawater.height && seawater.height<1){
continue;
}
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=5 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = city
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
}
}
return rets;
}
def getPublicDate(def pDate){
if(pDate>=20181001 && pDate<20200201){
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
}
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
return pdate;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
private String csvHandlerStr(String str) {
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
String tempDescription=str;
//如果有逗号
if(str.contains(",")||str.contains(",")){
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
if(str.contains("\"")){
// tempDescription=str.replace("\"", "\"\"");
tempDescription=str.replace("\"", "");
}
//在将逗号转义
tempDescription="\""+tempDescription+"\"";
}
return tempDescription;
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
/*
* 生成一个用不重复的ID
*/
def GenNonDuplicateID(randomLength){
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
}
def marge(){
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
def ids = new HashSet();
def index = 0
file.eachLine{it->
index++
if(index>1){
def arrs = it.split("\\|@\\|")
def id = arrs[1]
if(!ids.contains(id)){
ids << id
}
}
}
println index
println ids.size()
}
}
@@ -0,0 +1,108 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CcbCommonUtils {
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def findCityCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def count = dbcursor.count()
if(count >1){
def data = null
for(def db : dbcursor){
if(db.city_code){
data = db
break
}
}
return data;
}
}
public static def findAllDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
map.put(db.fangjia_district, db.district_code)
}
return map;
}
public static def findPersonPrice(city, date){
def q = new BasicDBObject()
q.put("city", city)
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
public static def findTradesDetail(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
}
return lists
}
}
@@ -0,0 +1,100 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
/**
* 建行模型估值项目
* 城市数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class CityBatch extends BaseBatch{
static main(args) {
runBatch(new CityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def index = 1
for(def city : citys){
def data = CcbCommonUtils.findCityCodeValue(city)
def map = [:]
map.name = city
if(data){
map.city_code = data.city_code
map.prov_code = data.province_code
}
def cityObj = findCityInfo(city)
if(cityObj){
map.city_spell = cityObj.spell
map.city_level = cityObj.city_grade
map.seq = index
map.ifcapital = isCapital(cityObj.province, city)
map.ifmunicipal = ifMunicipal(city)
}
index++
rows << map
}
def allDatas = createDatas(rows)
def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"]
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false);
}
def createDatas(rows){
def allRows = []
rows.each { m ->
def data = []
data << m.city_code
data << m.name
data << m.city_spell
data << m.prov_code
data << m.city_code
data << m.city_level
data << m.seq
data << CcbCodeUtils.parseFlag(m.ifcapital)
data << CcbCodeUtils.parseFlag(m.ifmunicipal)
data << ""
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "fjw"
allRows << data
}
return allRows;
}
def findCityInfo(city){
def q = new BasicDBObject()
q.put("display", city)
return CcbCommonUtils.SERVER_AREA_COLL.findOne(q)
}
def isCapital(def prov, def city){
def provCity = CcbCommonUtils.provCityMap.get(prov)
if(provCity && provCity==city){
return "是"
}
return "否"
}
def ifMunicipal(def city){
if(["上海", "北京", "天津", "重庆"].contains(city)){
return "是"
}
return "否"
}
}
@@ -0,0 +1,64 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
/**
* 建行模型估值项目
* 估价系数数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class EvaluateRateBatch extends BaseBatch{
static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver")
static main(args) {
runBatch(new EvaluateRateBatch(), args);
}
@Override
public void process(Object args) {
def allDatas = createDatas(findEvaluationRate())
def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false);
}
def createDatas(def datas){
def allRows = []
datas.each { m ->
def data = []
data << m.id.toString()
data << m.rate_type
data << m.rate
data << m.rank
data << "全国"
data << m.rank_remark
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
allRows << data
}
return allRows
}
private def findEvaluationRate(){
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'";
def rets = []
sqlInstance.eachRow(sql,{
rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated]
});
return rets;
}
}
@@ -0,0 +1,99 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 建行模型估值项目
* 人工指导价同步
*/
@BatchOwner(Watcher.GaoJuan)
class GuiDingPriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
runBatch(new GuiDingPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets)
println city + " " + rets.size()
}
}
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false);
def rets = []
rets << headers
rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(dist){
def ret = []
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
if(dist.region){
ret << distCodeMap.get(dist.region)
}else{
ret << ""
}
ret << dist.region
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << ps.guiding_person
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def getDistrictMap(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
}
@@ -0,0 +1,85 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 分期数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class StagingDistrictBatch extends BaseBatch{
static main(args) {
runBatch(new StagingDistrictBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"])
def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap)
rows.addAll(rets)
println city +" 分期量:"+rets.size()
}
def results =[]
def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
results << headers
results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8");
}
def createCityData(def dists, def fenQiMap){
def rets = []
for(def d : dists){
if(d.master_district_name && d.master_district_name!="master"){
def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name)
if(masterDist){
def ret = []
ret << new ObjectId().toString()
ret << d._id.toString()
ret << masterDist._id.toString()
ret << ""
ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
// else{
// def udata = new BasicDBObject()
// udata.put("master_district_name", null)
// udata.put("m_date", new Date())
// println d.city +" "+d.name+" "+d._id.toString()
// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata))
// }
}
}
return rets;
}
def getFenQiMap(def dists){
def fenQiMap = [:]
for(def d : dists){
if(d.master_district_name){
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
}
return fenQiMap;
}
}
@@ -0,0 +1,21 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys) {
def data = CcbCommonUtils.findCityCodeValue(city)
def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
def count = 0
sqlInstance.execute(sql,{
count = it.count
});
println city +" "+count
}
@@ -0,0 +1,177 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 建行模型估值项目
* 第三方均价数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
runBatch(new ThirdPartyPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def index = 0
for(def city : citys){
index++
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"])
def thirdPriceMap = findThirdPartyPrice(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
rows.addAll(rets)
println index+" "+city+" "+rets.size()
}
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false);
}
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def rets = []
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def ret = []
ret << third._id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
if(d.region){
ret << distCodeMap.get(d.region)
}else{
ret << ""
}
ret << d.region
ret << third.s_date
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
}else{
ret << ""
}
if(third.anjuke_esf_price){
ret << third.anjuke_esf_price
}else{
ret << ""
}
if(third.soufang_esf_price){
ret << third.soufang_esf_price
}else{
ret << ""
}
if(third.lianjia_esf_price){
ret << third.lianjia_esf_price
}else{
ret << ""
}
if(third.cityhouse_esf_price){
ret << third.cityhouse_esf_price
}else{
ret << ""
}
if(third.maitian_esf_price){
ret << third.maitian_esf_price
}else{
ret << ""
}
if(third.beike_esf_price){
ret << third.beike_esf_price
}else{
ret << ""
}
if(third.get("365house_esf_price")){
ret << third.get("365house_esf_price")
}else{
ret << ""
}
if(third.zhongyuan_esf_price){
ret << third.zhongyuan_esf_price
}else{
ret << ""
}
if(third.tuboshi_esf_price){
ret << third.tuboshi_esf_price
}else{
ret << ""
}
if(third.qfang_esf_price){
ret << third.qfang_esf_price
}else{
ret << ""
}
if(third.fangtan_esf_price){
ret << third.fangtan_esf_price
}else{
ret << ""
}
if(third.fangduoduo_esf_price){
ret << third.fangduoduo_esf_price
}else{
ret << ""
}
if(third.get("5i5j_esf_price")){
ret << third.get("5i5j_esf_price")
}else{
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
return rets;
}
def findThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", curDate)
def dbcursor = AVERAGE_STATISTICS.find(q)
def map = [:]
for(def db : dbcursor){
map.put(db.region+"##"+db.name, db)
}
return map
}
}
@@ -74,20 +74,20 @@ class AddThirdPriceDistrict {
static main(args) {
def task = new AddThirdPriceDistrict();
// task.run()
task.run()
def mapName = "锦华星光苑3,4幢"
// def mapName = "锦华星光苑3,4幢"
//// println task.pattern("港新花苑A1区");
println mapName.replaceAll("\\w+-?\\w+(幢)", "")
println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
// println mapName.replaceAll("\\w+-?\\w+(幢)", "")
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
}
def run(){
// exportDatas();
exportDatas();
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// addInfo();
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
}
def addCrawlMapInfo( def path){
@@ -135,29 +135,32 @@ class AddThirdPriceDistrict {
def exportDatas(){
def city = "无锡"
def datas = findMatchErrorDatas(city);
println "共【"+datas.size()+"】条"
datas = filterName(datas);
println "过滤后共【"+datas.size()+"】条"
def resMap = groupByName(datas);
def rets = []
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
resMap.each {k,v->
def ret = []
def arrs = k.split("##")
ret << arrs[0]
ret << arrs[1]
if(arrs.size()==3 && arrs[2]){
ret << arrs[2]
}else{
ret << ""
//跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞",
def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
for(def city : citys){
def datas = findMatchErrorDatas(city);
println "共【"+datas.size()+"】条"
datas = filterName(datas);
println "过滤后共【"+datas.size()+"】条"
def resMap = groupByName(datas);
def rets = []
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
resMap.each {k,v->
def ret = []
def arrs = k.split("##")
ret << arrs[0]
ret << arrs[1]
if(arrs.size()==3 && arrs[2]){
ret << arrs[2]
}else{
ret << ""
}
ret << resAddrMap.get(k)
ret << v.join(",")
rets << ret
}
ret << resAddrMap.get(k)
ret << v.join(",")
rets << ret
ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后")
}
ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后")
}
def filterName(datas){
@@ -465,7 +468,7 @@ class AddThirdPriceDistrict {
def findMatchErrorDatas(def city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date",202005)
q.put("s_date",new BasicDBObject('$gte', 202101))
// q.put("name","博济聚创科技园")
// q.put("source","贝壳找房")
def dbcursor = matchErrorLogColl.find(q)
@@ -7,7 +7,7 @@ import java.sql.ResultSet
class EsSearchUtils {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -394,6 +394,7 @@ class FiledStrand {
}
return estateCharege;
}
static def processHouseholdCount(def household_count, def estateType2){
if(household_count){
household_count = household_count.toInteger()
@@ -56,11 +56,11 @@ class GaoDeMapCrawl extends BaseBatch{
// def prefix = "road_"
// def categorys = ["190301"]//道路相关
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
def prefix = "village_"
def categorys = ["190108","190109"]//村庄
def prefix = "company_"
def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
// def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
def totalCount = new AtomicLong()
static main(args) {
@@ -118,7 +118,8 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys =["无锡"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
@@ -29,6 +29,8 @@ class OpenApiUtils {
rMap.region = dto.district
rMap.name = dto.name
rMap.id = dto.id
rMap.lng = dto.lng
rMap.lat = dto.lat
if(dto.address && dto.address!="null"){
rMap.address = dto.address
}
@@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
class WuXiAddressTest {
static main(args) {
def task = new WuXiAddressTest()
task.run()
//task.run()
task.addAddress()
}
def addAddress(){
ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d->
def prov = d['C']
def city = d['D']
def region = d['E']
def address = d['B']
def road = d['J']
def street = d['R']
def newAddress = prov+"省"+city +"市"
if(region == "惠山"){
newAddress = newAddress + region +"区"
}else{
newAddress = newAddress + region +"市"
}
d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "")
def dist = d['U']
def s1 = null
if(road){
s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","")
newAddress = newAddress + street + dist + road + s1
}else if(d['L']){
s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","")
newAddress = newAddress + street + dist + d['L'] + s1
}
d['V'] = newAddress
},0)
}
def run(){
@@ -18,6 +51,8 @@ class WuXiAddressTest {
//
// }
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
if(!d['S']){
def locationStr = getLocationInfo(d['D'], d['Q'])
@@ -47,11 +47,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-05-01"
static def statDate = "2021-06-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -7,6 +7,13 @@ import java.util.regex.Pattern
class CcbCodeUtils {
//否:0 是:1
static def parseFlag(String flag){
if(flag && flag == "是"){
return 1;
}
return 0;
}
static def parseBuildYear(String buildDate) {
if(buildDate){
@@ -53,6 +60,44 @@ class CcbCodeUtils {
return null
}
// 高层塔楼 0101 增加“超高层” 0108
// 高层板楼 0102 增加“老洋房” 0109
// 小高层板楼 0103 增加“花园洋房” 0110
// 多层板楼 0104 增加“四合院” 0111
// 独栋别墅 0105 增加“老式别墅洋房” 0112
// 联排别墅 0106
// 叠拼别墅 0107
// 其他建筑形式 0199
static def parseBuildType(def buildType){
if(buildType){
if(buildType.contains("老式别墅洋房")){
return "0112"
}else if(buildType.contains("四合院")){
return "0111"
}else if(buildType.contains("花园洋房")){
return "0110"
}else if(buildType.contains("老洋房")){
return "0109"
}else if(buildType.contains("叠拼")){
return "0107"
}else if(buildType.contains("联排")){
return "0106"
}else if(buildType.contains("独栋")){
return "0105"
}else if(buildType.contains("多层")){
return "0104"
}else if(buildType.contains("超高层")){
return "0108"
}else if(buildType.contains("小高层")){
return "0103"
}else if(buildType.contains("高层")){
return "0102"
}
}
return "0199"
}
//朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399
static def getDirectionCode(def direction){
@@ -161,4 +206,19 @@ class CcbCodeUtils {
}
return 0;
}
//房屋用途
static def parseHouseUse(String houseUse){
if(houseUse){
if(houseUse.contains("住宅")){
return 11;
}else if(houseUse.contains("商住")){
return 13;
}else{
return 99
}
}
return 99
}
}
@@ -3,13 +3,13 @@ package com.fangjia.script.gj.ccb
import org.apache.commons.lang.StringUtils
class FileUtils {
static def batchCreateFile(ret, fileName, headers,charset){
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
File file = new File(fileName);
if (file.exists()) {
file.delete();
}
// if (file.exists()) {
// file.delete();
// }
OutputStream out = new FileOutputStream(fileName);
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
sb.append(headers.join("|@|"))
@@ -20,18 +20,18 @@ class FileUtils {
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(System.getProperty("line.separator"));
println index
if(row){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(System.getProperty("line.separator"));
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
@@ -0,0 +1,174 @@
package com.fangjia.script.gj.data
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
class BestDistrictPriceExport {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new BestDistrictPriceExport();
task.run()
}
def run(){
def city = "佛山"
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
def rets = []
rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"]
def distMap = findCreditHightDistrict2(city, "20210711");
distMap.each{region, dists->
if(region=="三水"){
def addIndex = 0
for(def dist : dists){
if(addIndex <=30){
def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name)
def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711")
if(otherPrice){
def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def ret = []
ret << "佛山"
ret << dist.region
ret << dist.district_name
ret << dist.estate_type
for(def weekend : months){
def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend)
if(obj){
ret << obj.avg_price
}else{
ret << ""
}
}
ret << otherPrice.lianjia_esf_price
ret << stat.credit
rets << ret
addIndex++
}
}
}
}
}
}
ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格")
}
def findDistrictPrice(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
def obj = DIST_SELL_STAT.findOne(q)
return obj
}
def findCreditHightDistrict(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def typeMap = retMaps.get(d.region)
if(!typeMap){
typeMap = [:]
retMaps.put(d.region, typeMap)
}
def rets = typeMap.get(d.estate_type)
if(!rets){
rets = []
typeMap.put(d.estate_type, rets)
}
def count = 0
typeMap.each {k,v->
if(v){
count = count + v.size()
}
}
if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){
def m = [:]
m.city = d.city
m.region = d.region
m.district_name = d.district_name
rets << m
}else if(rets.size() <= 20){
def m = [:]
m.city = d.city
m.region = d.region
m.district_name = d.district_name
rets << m
}
def total = 0
retMaps.each {k,m->
m.each {t,v->
total = total + v.size()
}
}
if(total>100){
break;
}
}
def rows = []
retMaps.each {k,m->
m.each {t,v->
rows.addAll(v)
}
}
return rows
}
def findCreditHightDistrict2(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
def m = [:]
m.city = d.city
m.region = d.region
m.estate_type = d.estate_type
m.district_name = d.district_name
rets << m
}
//
// def rows = []
// retMaps.each {k,v->
// rows.addAll(v)
// }
return retMaps;
}
def findNearThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", 202106)
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
return AVERAGE_STATISTICS.findOne(q)
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"]
def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"]
def rets = []
rets << headers
for(def s : districts){
def arrs = s.split("_")
def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2])
def ret = []
ret << s
if(m){
ret << m.city
ret << m.district
ret << m.name
if(m.alias){
ret << m.alias.toString().replace("[","").replace("]","")
}else{
ret << ""
}
ret << m.address
if(m.addressAlias){
ret << m.addressAlias.toString().replace("[","").replace("]","")
}else{
ret << ""
}
ret << m.avgPrice
ret << m.lng
ret << m.lat
ret << m.block
ret << m.loopLine
ret << m.propertyType
ret << m.buildingType
ret << m.commercialArea
ret << m.totalUnitCount
ret << m.managementFee
ret << m.parking
ret << m.parkingDescription
ret << m.ownershipPeriod
ret << m.ownershipType
ret << m.startBuildTime
ret << m.startSellingTime
ret << m.finishBuildTime
ret << m.moveinTime
ret << m.floorSpace
ret << m.constructionSpace
ret << m.plotRatio
ret << m.greenPercentage
ret << m.preSalePermit
ret << m.developer
ret << m.propertyManagement
ret << m.decorationStandard
ret << m.buildingStructure
ret << m.supportingFacilities
ret << m.transportation
ret << m.relatedInfo
ret << m.description
ret << m.salesOfficeAdd
ret << m.salesPhoneNumber
ret << m.discount
ret << m.buildingCount
ret << m.finishYear
}
rets << ret
}
ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据")
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -10,22 +11,23 @@ import com.mongodb.DBCollection
class RegionPriceExport {
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPriceExport();
task.run()
}
def run(){
def months = [202010,202011,202012,202101,202102,202103]
def regions = CommonUtils.getAllRegions("福州","region")
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
def regions = CommonUtils.getAllRegions("佛山","region")
def rets = []
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
regions.each{region,values->
def data = []
data << "福州"
data << "佛山"
data << region
for(def month :months){
def dbo = findPrice("福州", region, month);
def dbo = findWeekendPrice("佛山", region, month);
if(dbo){
data << dbo.avg_price
}else{
@@ -34,7 +36,7 @@ class RegionPriceExport {
}
rets << data
}
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格")
}
def findPrice(city, region, month){
@@ -45,4 +47,14 @@ class RegionPriceExport {
def dbo = REGION_SELL_STAT.findOne(q)
return dbo
}
def findWeekendPrice(city, region, month){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("cat", "区域")
q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
def dbo = REGION_STAT.findOne(q)
return dbo
}
}
@@ -1,6 +1,8 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
@@ -13,7 +15,7 @@ class ExportData {
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -24,8 +26,8 @@ class ExportData {
static main(args) {
def task = new ExportData();
task.exportThird()
// task.exportMap();
// task.exportThird()
task.exportMap();
}
def exportMap(){
@@ -51,19 +53,18 @@ class ExportData {
//"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特"
//宁夏 ----- 中卫 固原 石嘴山 吴忠 银川
//青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁
def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
"中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
"防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
//"沈阳","大连","抚顺","鞍山"
//"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆"
//"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛",
//"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州",
//"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城",
// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"];
for(def city : citys){
def path = "D:\\数据\\小区\\地图\\${city}.xlsx";
def datas = findAllDatas(city, null)
println "${city}共计:"+datas.size()
ExcelUtils.write(path, datas, "数据")
}
def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx";
def datas = findAllDatas(citys, null)
println "共计:"+datas.size()
ExcelUtils.write(path, datas, "数据")
}
@@ -98,9 +99,9 @@ class ExportData {
def findAllDatas(def citys, def pname){
def findAllDatas(def city, def pname){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1")
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'")
// sqlBuffer.append(" and pname = '"+pname+"'")
// sqlBuffer.append(" and cityname = '"+pname+"'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
@@ -113,16 +114,19 @@ class ExportData {
if(source && source=="高德"){
println index
def type = rs.getObject("type")
def city = rs.getString("cityname")
if(citys.contains(city) && type.contains("商务住宅")){
//def city = rs.getString("cityname")
if(type.contains("住宅")){
def ret = []
ret << city
ret << rs.getString("adname")
ret << rs.getString("name")
ret << rs.getString("address")
ret << rs.getString("type")
ret << rs.getFloat("location.lon")
ret << rs.getFloat("location.lat")
def loaction = rs.getObject("location")
if(loaction){
ret << DataHandleUtils.getDoubleRound(6, loaction.x)
ret << DataHandleUtils.getDoubleRound(6, loaction.y)
}
rets << ret
}
}
@@ -68,13 +68,16 @@ class FangjiaClientUtils {
public static def addressStand(def address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
def content = new HttpRequest().newGet(url, null, "utf-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
@@ -1,9 +1,12 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import java.util.regex.Pattern
@@ -20,20 +23,13 @@ class MapDistrictProcess {
* @return
*/
def run(){
//"江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx","安徽地图.xlsx",
//"上海地图.xlsx","北京地图.xlsx"
//"安徽地图.xlsx","江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx",
//"重庆地图.xlsx","天津地图.xlsx",
//"上海地图.xlsx","北京地图.xlsx",
//"河北地图.xlsx","河南地图.xlsx","湖北地图.xlsx","湖南地图.xlsx","广西地图.xlsx",
//"吉林地图.xlsx","辽宁地图.xlsx""辽宁地图.xlsx","吉林地图.xlsx","黑龙江地图.xlsx","山西地图.xlsx","海南地图.xlsx"]
def files = ["青海地图.xlsx","宁夏地图.xlsx","内蒙古地图.xlsx","云南地图.xlsx","陕西地图.xlsx","新疆地图.xlsx","四川地图.xlsx"]
def files = ["武汉.xlsx"]
for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
typeFilter(filePath)
nameStandard(filePath);
dataFilter(filePath)
// typeFilter(filePath)
// nameStandard(filePath);
// dataFilter(filePath)
matchDistDatas(filePath)
houseCalculate(filePath);
////// //统计是否存在多个区域
@@ -114,15 +110,24 @@ class MapDistrictProcess {
def matchDistDatas(def path){
def index = 0
ExcelUtils.multiThreadingModify(path,{ List row ->
ExcelUtils.modify(path,{ List row ->
index ++
if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],);
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
row['J'] = rMap.city
row['K'] = rMap.region
row['L'] = rMap.name
row['M'] = rMap.address
if(rMap.lat){
def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
def maplng = baidu.x
def maplat = baidu.y
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
row['Y'] = score
}
}
println index +" "+row
}
@@ -26,15 +26,18 @@ class SeaweedDataMatchProcess {
}
}
if(!rMap && address){
// if (findPattern(address)){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
// }
}
if(!rMap && name){
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
}
if(!rMap && address){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
if(rMap && rMap.region && !region.contains(rMap.region)){
rMap = null
}
}
if(rMap && rMap.id && region.contains(rMap.region)) {
@@ -0,0 +1,265 @@
package com.fangjia.script.gj.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.lucene.queryparser.classic.QueryParser
class SourceDistDataMatch {
static main(args) {
def task = new SourceDistDataMatch();
// task.run()
task.mapRun()
// task.filterName()
// task.filterType()
}
def run(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(obj){
d['R'] = 1
}else{
def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709)
if(distCount){
d['W'] = distCount
}
}
},0)
}
def mapRun(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['N'] && !d['F']){
def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']]
def locationMap = matchMap(dist)
if(locationMap){
d['F'] = locationMap.source
d['G'] = locationMap.region
d['H'] = locationMap.mapName
d['I'] = locationMap.address
d['J'] = locationMap.type
d['K'] = locationMap.lng
d['L'] = locationMap.lat
d['M'] = locationMap.score
}
}
},0)
}
def filterName(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['N']) {
def name = d['C']
if (name) {
def flag = NameProcess.filterName(name)
if (!flag) {
d['N'] = 4
}
}
}
},0)
}
def filterType(){
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['N']){
def type = d['J']
if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){
}else if(type && type.contains("商务住宅") && !type.contains("商住两用")){
def count = 0
if(d['S'] && d['S']!='null'){
count = d['S'].toDouble().toInteger()
}
if(count<10){
d['N'] = 3
}
}else{
if(type){
d['N'] = 3
}
}
}
},0)
}
def matchMap(def dist){
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
if(!gaodeRest && dist.address){
gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德")
}
def gdlocationMap = null
def bdlocationMap = null
if(gaodeRest){
gdlocationMap = getMapLocation2(dist, gaodeRest);
if(gdlocationMap){
gdlocationMap.source = "高德"
}
}
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
if(baiduRest){
bdlocationMap = getMapLocation2(dist, baiduRest);
if(bdlocationMap){
bdlocationMap.source = "百度"
}
}
if(gdlocationMap && bdlocationMap){
if(gdlocationMap.score>=bdlocationMap.score){
return gdlocationMap
}else{
return bdlocationMap
}
}else if(gdlocationMap){
return gdlocationMap
}else if(bdlocationMap){
return bdlocationMap
}
return null
}
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
def aliasStr = null;
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
def getMapLocation2(def dist, def rets){
def data1 = findSimilarData(dist, rets, true)
def data2 = findSimilarData(dist, rets, false)
println data1
println data2
if(data1 && data1.score>1){
return data1
}else if(data2 && data2.score>1){
return data2
}
return null
}
def findSimilarData(def dist, def rets, def regionImpr){
def lat = null
def lng = null
def region = null
def mapName = null
def address = null
def score = 0
def type = null
def maxScore = 0
def standardName = null
for(def r : rets){
if(!r.city){
continue
}
if(!r.city.contains(dist.city)){
continue
}
if(["公交车站"].contains(r.type)){
continue
}
if(regionImpr && !r.district){
continue
}
if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){
continue
}
def name = nameReplace(r.name, dist.city)
def distName = nameReplace(dist.name, dist.city)
def score3 = 0.0
if(r.district && dist.region && r.district.contains(dist.region)){
score3 = 0.2
}
def score1 = 0.0;
def score2 = 0.0
if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){
if(r.address){
score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2
}
}else{
score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2;
if(dist.address && r.address){
score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2
}
}
score3 = score1 + score2
if(r.type && r.type.contains("住宅")){
score3 = score3 + 0.2
}
if(score3 >maxScore){
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
standardName = name
address = r.address
type = r.type
score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10)
maxScore = score3
}
println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3
}
if(lng){
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName]
}
return null
}
def nameReplace(def s, city){
if(s.contains("(") && s.contains(")")){
def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")"))
if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){
s = s.toString().substring(0, s.indexOf("("))
}
}
def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim()
if(name.startsWith(city)){
def newName = name.replace(city+"市","").replace(city,"")
if(newName && newName=="花园"){
return name
}else{
name = newName
}
}
return name
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -49,7 +49,7 @@ class ZsPersonProcess {
// def loactionMap = task.matchMap(dist)
// println loactionMap
// task.matchMap()
task.matchMap()
// task.matchDb()
// task.matchDb2()
@@ -19,9 +19,9 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
// task.houseRun()
task.houseRun()
task.statHouseUpdateCount()
// task.statHouseUpdateCount()
}
def run(){
@@ -99,37 +99,35 @@ class EsMonthHouseStat {
}
def houseRun(){
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
def city = d['B']
if(city == "湖州"){
def count = findHouseCount(city, 20210301, 20210601)
d['E'] = count
ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d->
def city = d['A']
def count = findHouseCount(city, 20190709, 20210709)
d['E'] = count
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
alias = dist.alias
}else{
alias << dist.name
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['D'] = precent+"%"
def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['F'] = precent+"%"
},0)
}
@@ -0,0 +1,65 @@
package com.fangjia.script.gj.region
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class MapAreaUpdate {
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
static main(args) {
def task = new MapAreaUpdate()
task.run()
}
def run(){
def rets = findAllCity()
for(def city : rets){
def retMap = findCityAndProvCode(city)
if(retMap){
update(retMap)
}
}
}
def update(retMap){
def q = new BasicDBObject()
q.put("fangjia_city",retMap.fangjia_city)
q.put("district", "ÊÐÏ½Çø")
def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("city_code", retMap.city_code)
udata.put("province_code", retMap.province_code)
FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false)
}
}
def findCityAndProvCode(def city){
def q = new BasicDBObject()
q.put("fangjia_city",city)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def map = [:]
for(def d : dbcursor){
if(d.city_code && d.province_code && d.district_code){
map.put("fangjia_city",d.fangjia_city)
map.put("city_code",d.city_code)
map.put("province_code",d.province_code)
break
}
}
return map;
}
def findAllCity(){
def q = new BasicDBObject()
q.put("fangjia_district",null)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def citys = []
for(def d : dbcursor){
citys << d.fangjia_city
}
citys.unique()
return citys
}
}
@@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{
@Override
void process(Object args) {
citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""])
citys.addAll(["¸£ÖÝ"])
// getNoDataCitys();
def cityurl = "http://www.zhuge.com/areachange/"
def cityUrls = getCityUrlCrawal(cityurl);
@@ -140,6 +140,7 @@ public class OpenApiClient {
return null
}
public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) {
if (StringUtils.isBlank(input)) {