add ccb batch file
This commit is contained in:
+154
@@ -0,0 +1,154 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 小区数据同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists)
|
||||
rows.addAll(rets)
|
||||
println city +" "+rets.size()
|
||||
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
allDatas << headers
|
||||
allDatas.addAll(rows)
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
|
||||
def rets = []
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets << createData(cityCodeDatas, distCodeMap, dist)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(def cityCodeDatas, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
ret << cityCodeDatas.city_code
|
||||
if(dist.region){
|
||||
ret << distCodeMap.get(dist.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist._id.toString()
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "物业小区"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 区域数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDistrictBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataDistrictBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def datas = findDistrictCodeValue(city)
|
||||
if(datas){
|
||||
rows.addAll(datas)
|
||||
}
|
||||
println city+" "+ datas.size()
|
||||
}
|
||||
|
||||
def allDatas = createDatas(rows)
|
||||
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8");
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
def allRows = []
|
||||
def index = 1
|
||||
rows.each { m ->
|
||||
def data = []
|
||||
data << m.dist_code
|
||||
data << m.dist_name
|
||||
data << m.dist_spell
|
||||
data << m.city_id
|
||||
data << m.dist_code
|
||||
data << index
|
||||
data << ""
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "FJW"
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
index++
|
||||
allRows << data
|
||||
}
|
||||
return allRows;
|
||||
}
|
||||
|
||||
def findDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
q.put("district", new BasicDBObject('$ne',"市辖区"))
|
||||
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def lists = []
|
||||
for(def db : dbcursor){
|
||||
if(db.district_code){
|
||||
def m = [:]
|
||||
m.id = db._id.toString()
|
||||
m.dist_name = db.fangjia_district
|
||||
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
|
||||
m.city_id = db.city_code
|
||||
m.dist_code = db.district_code
|
||||
lists << m
|
||||
}
|
||||
}
|
||||
return lists;
|
||||
}
|
||||
}
|
||||
+520
@@ -0,0 +1,520 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataForsaleBatch extends BaseBatch{
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731);
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeDatas.city_code
|
||||
def regionCode = distCodeMap.get(rMap.region)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = rMap.region
|
||||
m.region_code = regionCode
|
||||
m.district_id = rMap.get("_id").toString()
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
}
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
println city + " 房源总量:"+retMap.size()
|
||||
def allDatas = createSellDatas(city, retMap.values());
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true);
|
||||
println "txt文件生成 "
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createSellDatas(def city, def values){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(values.size())
|
||||
println city +"总房源量:"+values.size()
|
||||
|
||||
def datas = Collections.synchronizedList(new ArrayList());
|
||||
for(def i=0;i<values.size();i++){
|
||||
final def m = values[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
datas << converData(m)
|
||||
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return datas ;
|
||||
}
|
||||
|
||||
def converData(def m){
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
data << m.region
|
||||
data << m.district_id
|
||||
data << m.district_name
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.address==null?"":m.address==null
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << ""
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
data << m.hall==null?"":m.hall
|
||||
data << ""
|
||||
data << m.toilet==null?"":m.toilet
|
||||
data << m.direction_code==null?"":m.direction_code
|
||||
data << m.fitment_code==null?"":m.fitment_code
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.p_date==null?"":m.p_date
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.root
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.c_date==null?"":m.c_date
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
data << ""
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
if(seawater.height && seawater.height<1){
|
||||
continue;
|
||||
}
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=5 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(rootMap.get(seawater.root)){
|
||||
rowMap.root = rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
|
||||
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String csvHandlerStr(String str) {
|
||||
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
String tempDescription=str;
|
||||
//如果有逗号
|
||||
if(str.contains(",")||str.contains(",")){
|
||||
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if(str.contains("\"")){
|
||||
// tempDescription=str.replace("\"", "\"\"");
|
||||
tempDescription=str.replace("\"", "");
|
||||
}
|
||||
//在将逗号转义
|
||||
tempDescription="\""+tempDescription+"\"";
|
||||
}
|
||||
return tempDescription;
|
||||
}
|
||||
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
}
|
||||
+521
@@ -0,0 +1,521 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataForsaleBatch2 extends BaseBatch{
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch2(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def distMap = getDistrictMap(city)
|
||||
exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731);
|
||||
}
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
if(d.alias){
|
||||
d.alias.each{name->
|
||||
if(!distMap.containsKey(d.region+"##"+name)){
|
||||
distMap.put(d.region+"##"+name, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){
|
||||
println city +" 小区key:"+distMap.size()
|
||||
def cityCode = cityCodeDatas.city_code
|
||||
def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap)
|
||||
def allDatas = createSellDatas(city, retMap);
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true);
|
||||
println "txt文件生成 "
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createSellDatas(def city, def map){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(map.size())
|
||||
println city +"总房源量:"+map.size()
|
||||
|
||||
def datas = []
|
||||
map.each{k,m->
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
datas << converData(m)
|
||||
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return datas ;
|
||||
}
|
||||
|
||||
def converData(def m){
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
data << m.region
|
||||
data << m.district_id
|
||||
data << m.district_name
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.address==null?"":m.address==null
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << ""
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
data << m.hall==null?"":m.hall
|
||||
data << ""
|
||||
data << m.toilet==null?"":m.toilet
|
||||
data << m.direction_code==null?"":m.direction_code
|
||||
data << m.fitment_code==null?"":m.fitment_code
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.p_date==null?"":m.p_date
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.root
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.c_date==null?"":m.c_date
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
data << ""
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
def retMap = new ConcurrentHashMap()
|
||||
println city +" 没过滤前:"+ houses.size()
|
||||
if(houses){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(houses.size())
|
||||
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
final def map = houses[k];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def rowMap = converData(distMap, map, cat)
|
||||
if(retMap){
|
||||
retMap.put(rowMap.houseId, rowMap)
|
||||
println totalCount.get() +" [匹配success]"+city+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
}else{
|
||||
println totalCount.get() +" [匹配fail]"+city+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
}
|
||||
latch.await()
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def converData(distMap, map, cat){
|
||||
def rMap = distMap.get(map.region +"##"+map.district_name)
|
||||
if(!rMap){
|
||||
return null
|
||||
}
|
||||
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
return null
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
return null
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
if(seawater.height && seawater.height<1){
|
||||
return null
|
||||
}
|
||||
if(seawater.height && seawater.height>60){
|
||||
return null
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=5 || seawater.room<1){
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
return null
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
return null
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(rootMap.get(seawater.root)){
|
||||
rowMap.root = rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
|
||||
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
def regionCode = distCodeMap.get(rMap.region)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
rowMap.city_code = cityCode
|
||||
rowMap.region = rMap.region
|
||||
rowMap.region_code = regionCode
|
||||
rowMap.district_id = rMap.get("_id").toString()
|
||||
rowMap.district_name = rMap.name
|
||||
rowMap.build_year = buildYear;
|
||||
rowMap.proptype = housePro
|
||||
rowMap.propyears = propertyYear;
|
||||
rowMap.estate_type = estateType
|
||||
rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
rowMap.address = rMap.address
|
||||
rowMap.build_structure_code = struCode
|
||||
rowMap.house_structure_code = null//getHouseStructureCode();
|
||||
rowMap.source = "FJW"
|
||||
return rowMap
|
||||
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String csvHandlerStr(String str) {
|
||||
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
String tempDescription=str;
|
||||
//如果有逗号
|
||||
if(str.contains(",")||str.contains(",")){
|
||||
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if(str.contains("\"")){
|
||||
// tempDescription=str.replace("\"", "\"\"");
|
||||
tempDescription=str.replace("\"", "");
|
||||
}
|
||||
//在将逗号转义
|
||||
tempDescription="\""+tempDescription+"\"";
|
||||
}
|
||||
return tempDescription;
|
||||
}
|
||||
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
|
||||
// public static def citys = ["北京"]
|
||||
|
||||
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
|
||||
|
||||
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static def findCityCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def count = dbcursor.count()
|
||||
if(count >1){
|
||||
def data = null
|
||||
for(def db : dbcursor){
|
||||
if(db.city_code){
|
||||
data = db
|
||||
break
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
public static def findAllDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
q.put("district", new BasicDBObject('$ne',"市辖区"))
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
map.put(db.fangjia_district, db.district_code)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public static def findPersonPrice(city, date){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)));
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 城市数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class CityBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new CityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def index = 1
|
||||
for(def city : citys){
|
||||
def data = CcbCommonUtils.findCityCodeValue(city)
|
||||
def map = [:]
|
||||
map.name = city
|
||||
if(data){
|
||||
map.city_code = data.city_code
|
||||
map.prov_code = data.province_code
|
||||
}
|
||||
def cityObj = findCityInfo(city)
|
||||
if(cityObj){
|
||||
map.city_spell = cityObj.spell
|
||||
map.city_level = cityObj.city_grade
|
||||
map.seq = index
|
||||
map.ifcapital = isCapital(cityObj.province, city)
|
||||
map.ifmunicipal = ifMunicipal(city)
|
||||
}
|
||||
index++
|
||||
rows << map
|
||||
}
|
||||
|
||||
def allDatas = createDatas(rows)
|
||||
def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8");
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
def allRows = []
|
||||
rows.each { m ->
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.name
|
||||
data << m.city_spell
|
||||
data << m.prov_code
|
||||
data << m.city_code
|
||||
data << m.city_level
|
||||
data << m.seq
|
||||
data << CcbCodeUtils.parseFlag(m.ifcapital)
|
||||
data << CcbCodeUtils.parseFlag(m.ifmunicipal)
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "fjw"
|
||||
allRows << data
|
||||
}
|
||||
return allRows;
|
||||
}
|
||||
|
||||
def findCityInfo(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("display", city)
|
||||
return CcbCommonUtils.SERVER_AREA_COLL.findOne(q)
|
||||
}
|
||||
|
||||
def isCapital(def prov, def city){
|
||||
def provCity = CcbCommonUtils.provCityMap.get(prov)
|
||||
if(provCity && provCity==city){
|
||||
return "是"
|
||||
}
|
||||
return "否"
|
||||
}
|
||||
|
||||
def ifMunicipal(def city){
|
||||
if(["上海", "北京", "天津", "重庆"].contains(city)){
|
||||
return "是"
|
||||
}
|
||||
return "否"
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 估价系数数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class EvaluateRateBatch extends BaseBatch{
|
||||
|
||||
static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new EvaluateRateBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def allDatas = createDatas(findEvaluationRate())
|
||||
def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def createDatas(def datas){
|
||||
def allRows = []
|
||||
datas.each { m ->
|
||||
def data = []
|
||||
data << m.id.toString()
|
||||
data << m.rate_type
|
||||
data << m.rate
|
||||
data << m.rank
|
||||
data << "全国"
|
||||
data << m.rank_remark
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "FJW"
|
||||
allRows << data
|
||||
}
|
||||
return allRows
|
||||
}
|
||||
|
||||
private def findEvaluationRate(){
|
||||
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'";
|
||||
def rets = []
|
||||
sqlInstance.eachRow(sql,{
|
||||
rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated]
|
||||
});
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 人工指导价同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GuiDingPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false);
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(rows)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(dist){
|
||||
def ret = []
|
||||
ret << ps._id.toString()
|
||||
ret << dist._id.toString()
|
||||
ret << dist.name
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(dist.region){
|
||||
ret << distCodeMap.get(dist.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << dist.region
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.guiding_value
|
||||
ret << ps.guiding_person
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 分期数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class StagingDistrictBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new StagingDistrictBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"])
|
||||
def fenQiMap = getFenQiMap(dists);
|
||||
def rets = createCityData(dists, fenQiMap)
|
||||
rows.addAll(rets)
|
||||
println city +" 分期量:"+rets.size()
|
||||
}
|
||||
|
||||
|
||||
def results =[]
|
||||
def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
|
||||
results << headers
|
||||
results.addAll(rows)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8");
|
||||
|
||||
}
|
||||
|
||||
def createCityData(def dists, def fenQiMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
if(d.master_district_name && d.master_district_name!="master"){
|
||||
def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name)
|
||||
if(masterDist){
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << d._id.toString()
|
||||
ret << masterDist._id.toString()
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
// else{
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("master_district_name", null)
|
||||
// udata.put("m_date", new Date())
|
||||
// println d.city +" "+d.name+" "+d._id.toString()
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getFenQiMap(def dists){
|
||||
def fenQiMap = [:]
|
||||
for(def d : dists){
|
||||
if(d.master_district_name){
|
||||
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
}
|
||||
return fenQiMap;
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 第三方均价数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThirdPartyPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index++
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"])
|
||||
def thirdPriceMap = findThirdPartyPrice(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
|
||||
rows.addAll(rets)
|
||||
println index+" "+city+" "+rets.size()
|
||||
}
|
||||
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def ret = []
|
||||
ret << third._id.toString()
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(d.region){
|
||||
ret << distCodeMap.get(d.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << d.region
|
||||
ret << third.s_date
|
||||
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
ret << third.zhugezhaofang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
ret << third.anjuke_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
ret << third.soufang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.lianjia_esf_price){
|
||||
ret << third.lianjia_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.cityhouse_esf_price){
|
||||
ret << third.cityhouse_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.maitian_esf_price){
|
||||
ret << third.maitian_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.beike_esf_price){
|
||||
ret << third.beike_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.get("365house_esf_price")){
|
||||
ret << third.get("365house_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.zhongyuan_esf_price){
|
||||
ret << third.zhongyuan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.tuboshi_esf_price){
|
||||
ret << third.tuboshi_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
ret << third.qfang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangtan_esf_price){
|
||||
ret << third.fangtan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangduoduo_esf_price){
|
||||
ret << third.fangduoduo_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
ret << third.get("5i5j_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", curDate)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
map.put(db.region+"##"+db.name, db)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
class BestDistrictPriceExport {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new BestDistrictPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "佛山"
|
||||
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
def rets = []
|
||||
rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"]
|
||||
def distMap = findCreditHightDistrict2(city, "20210711");
|
||||
distMap.each{region, dists->
|
||||
|
||||
if(region=="三水"){
|
||||
def addIndex = 0
|
||||
for(def dist : dists){
|
||||
if(addIndex <=30){
|
||||
def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name)
|
||||
def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711")
|
||||
if(otherPrice){
|
||||
def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price
|
||||
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
|
||||
def ret = []
|
||||
ret << "佛山"
|
||||
ret << dist.region
|
||||
ret << dist.district_name
|
||||
ret << dist.estate_type
|
||||
for(def weekend : months){
|
||||
def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend)
|
||||
if(obj){
|
||||
ret << obj.avg_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
ret << otherPrice.lianjia_esf_price
|
||||
ret << stat.credit
|
||||
rets << ret
|
||||
addIndex++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
|
||||
def findDistrictPrice(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def obj = DIST_SELL_STAT.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def findCreditHightDistrict(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def typeMap = retMaps.get(d.region)
|
||||
if(!typeMap){
|
||||
typeMap = [:]
|
||||
retMaps.put(d.region, typeMap)
|
||||
}
|
||||
|
||||
def rets = typeMap.get(d.estate_type)
|
||||
if(!rets){
|
||||
rets = []
|
||||
typeMap.put(d.estate_type, rets)
|
||||
}
|
||||
|
||||
def count = 0
|
||||
typeMap.each {k,v->
|
||||
if(v){
|
||||
count = count + v.size()
|
||||
}
|
||||
}
|
||||
|
||||
if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}else if(rets.size() <= 20){
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}
|
||||
|
||||
def total = 0
|
||||
retMaps.each {k,m->
|
||||
m.each {t,v->
|
||||
total = total + v.size()
|
||||
}
|
||||
}
|
||||
if(total>100){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
def rows = []
|
||||
retMaps.each {k,m->
|
||||
m.each {t,v->
|
||||
rows.addAll(v)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
def findCreditHightDistrict2(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.estate_type = d.estate_type
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}
|
||||
//
|
||||
// def rows = []
|
||||
// retMaps.each {k,v->
|
||||
// rows.addAll(v)
|
||||
// }
|
||||
return retMaps;
|
||||
}
|
||||
|
||||
def findNearThirdPartyPrice(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", 202106)
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
return AVERAGE_STATISTICS.findOne(q)
|
||||
}
|
||||
}
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class SourceDistDataMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new SourceDistDataMatch();
|
||||
// task.run()
|
||||
task.mapRun()
|
||||
// task.filterName()
|
||||
// task.filterType()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(obj){
|
||||
d['R'] = 1
|
||||
}else{
|
||||
def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709)
|
||||
if(distCount){
|
||||
d['W'] = distCount
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def mapRun(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N'] && !d['F']){
|
||||
def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']]
|
||||
def locationMap = matchMap(dist)
|
||||
if(locationMap){
|
||||
d['F'] = locationMap.source
|
||||
d['G'] = locationMap.region
|
||||
d['H'] = locationMap.mapName
|
||||
d['I'] = locationMap.address
|
||||
d['J'] = locationMap.type
|
||||
d['K'] = locationMap.lng
|
||||
d['L'] = locationMap.lat
|
||||
d['M'] = locationMap.score
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def filterName(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N']) {
|
||||
def name = d['C']
|
||||
if (name) {
|
||||
def flag = NameProcess.filterName(name)
|
||||
if (!flag) {
|
||||
d['N'] = 4
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def filterType(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N']){
|
||||
def type = d['J']
|
||||
if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){
|
||||
|
||||
}else if(type && type.contains("商务住宅") && !type.contains("商住两用")){
|
||||
def count = 0
|
||||
if(d['S'] && d['S']!='null'){
|
||||
count = d['S'].toDouble().toInteger()
|
||||
}
|
||||
if(count<10){
|
||||
d['N'] = 3
|
||||
}
|
||||
}else{
|
||||
if(type){
|
||||
d['N'] = 3
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def matchMap(def dist){
|
||||
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
|
||||
if(!gaodeRest && dist.address){
|
||||
gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德")
|
||||
}
|
||||
def gdlocationMap = null
|
||||
def bdlocationMap = null
|
||||
if(gaodeRest){
|
||||
gdlocationMap = getMapLocation2(dist, gaodeRest);
|
||||
if(gdlocationMap){
|
||||
gdlocationMap.source = "高德"
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
if(baiduRest){
|
||||
bdlocationMap = getMapLocation2(dist, baiduRest);
|
||||
if(bdlocationMap){
|
||||
bdlocationMap.source = "百度"
|
||||
}
|
||||
}
|
||||
|
||||
if(gdlocationMap && bdlocationMap){
|
||||
if(gdlocationMap.score>=bdlocationMap.score){
|
||||
return gdlocationMap
|
||||
}else{
|
||||
return bdlocationMap
|
||||
}
|
||||
}else if(gdlocationMap){
|
||||
return gdlocationMap
|
||||
}else if(bdlocationMap){
|
||||
return bdlocationMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
def getMapLocation2(def dist, def rets){
|
||||
def data1 = findSimilarData(dist, rets, true)
|
||||
def data2 = findSimilarData(dist, rets, false)
|
||||
println data1
|
||||
println data2
|
||||
if(data1 && data1.score>1){
|
||||
return data1
|
||||
}else if(data2 && data2.score>1){
|
||||
return data2
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findSimilarData(def dist, def rets, def regionImpr){
|
||||
def lat = null
|
||||
def lng = null
|
||||
def region = null
|
||||
def mapName = null
|
||||
def address = null
|
||||
def score = 0
|
||||
def type = null
|
||||
def maxScore = 0
|
||||
def standardName = null
|
||||
for(def r : rets){
|
||||
if(!r.city){
|
||||
continue
|
||||
}
|
||||
if(!r.city.contains(dist.city)){
|
||||
continue
|
||||
}
|
||||
if(["公交车站"].contains(r.type)){
|
||||
continue
|
||||
}
|
||||
if(regionImpr && !r.district){
|
||||
continue
|
||||
}
|
||||
if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){
|
||||
continue
|
||||
}
|
||||
|
||||
def name = nameReplace(r.name, dist.city)
|
||||
|
||||
def distName = nameReplace(dist.name, dist.city)
|
||||
|
||||
def score3 = 0.0
|
||||
if(r.district && dist.region && r.district.contains(dist.region)){
|
||||
score3 = 0.2
|
||||
}
|
||||
|
||||
def score1 = 0.0;
|
||||
def score2 = 0.0
|
||||
if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){
|
||||
if(r.address){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2
|
||||
}
|
||||
}else{
|
||||
score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2;
|
||||
if(dist.address && r.address){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2
|
||||
}
|
||||
}
|
||||
|
||||
score3 = score1 + score2
|
||||
if(r.type && r.type.contains("住宅")){
|
||||
score3 = score3 + 0.2
|
||||
}
|
||||
if(score3 >maxScore){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
standardName = name
|
||||
address = r.address
|
||||
type = r.type
|
||||
score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10)
|
||||
maxScore = score3
|
||||
}
|
||||
println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3
|
||||
|
||||
}
|
||||
if(lng){
|
||||
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def nameReplace(def s, city){
|
||||
if(s.contains("(") && s.contains(")")){
|
||||
def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")"))
|
||||
if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){
|
||||
s = s.toString().substring(0, s.indexOf("("))
|
||||
}
|
||||
}
|
||||
def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim()
|
||||
if(name.startsWith(city)){
|
||||
def newName = name.replace(city+"市","").replace(city,"")
|
||||
if(newName && newName=="花园"){
|
||||
return name
|
||||
}else{
|
||||
name = newName
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.fangjia.script.gj.region
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class MapAreaUpdate {
|
||||
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new MapAreaUpdate()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findAllCity()
|
||||
for(def city : rets){
|
||||
def retMap = findCityAndProvCode(city)
|
||||
if(retMap){
|
||||
update(retMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def update(retMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city",retMap.fangjia_city)
|
||||
q.put("district", "ÊÐÏ½Çø")
|
||||
def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city_code", retMap.city_code)
|
||||
udata.put("province_code", retMap.province_code)
|
||||
FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false)
|
||||
}
|
||||
}
|
||||
|
||||
def findCityAndProvCode(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city",city)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.city_code && d.province_code && d.district_code){
|
||||
map.put("fangjia_city",d.fangjia_city)
|
||||
map.put("city_code",d.city_code)
|
||||
map.put("province_code",d.province_code)
|
||||
break
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def findAllCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_district",null)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def citys = []
|
||||
for(def d : dbcursor){
|
||||
citys << d.fangjia_city
|
||||
}
|
||||
citys.unique()
|
||||
return citys
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user