Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+6
-5
@@ -13,8 +13,9 @@ import com.mongodb.DBObject
|
||||
*/
|
||||
|
||||
class CityLatlngBox {
|
||||
public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
|
||||
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017)
|
||||
//public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
|
||||
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
|
||||
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020)
|
||||
|
||||
public static void main(String[] args) {
|
||||
CityLatlngBox sd = new CityLatlngBox();
|
||||
@@ -22,7 +23,7 @@ class CityLatlngBox {
|
||||
}
|
||||
|
||||
void process() {
|
||||
def cities = CommonUtils.findAllCity()
|
||||
def cities = ["白沙","阿坝州","甘孜州","阿勒泰","临沧","那曲","三沙","喀什","昌吉","迪庆州","普洱","德宏州","湘西","塔城","凉山州","屯昌","山南","海西州","阿拉善盟","乐东","定安","怒江州","黄山","图木舒克","楚雄","临高","乌兰察布","延边","文山州","锡林郭勒盟","陵水","天门","淮北","黔东南","万宁","湘潭","海北州","琼中","吐鲁番","惠州","巴音郭楞","恩施","铜仁","黔南","西双版纳州","潜江","林芝","文昌","仙桃","神农架","和田","黔西南","果洛州","儋州","哈密","大理","东方","日喀则","兴安盟","昌江","澄迈","保亭","济源","五指山","阿拉尔","伊犁","临夏","海南州","琼海","石河子","昌都","五家渠","玉树州","博尔塔拉","东营","铜陵","黄南州","廊坊","嘉峪关","马鞍山","宁波","广安","中山","克孜勒苏","厦门","莱芜","鄂州","东莞","海口","鹤壁","六盘水","漯河","三亚","珠海","深圳","乌海","淮南","北海","双鸭山","襄阳","新余","汕头","克拉玛依","鹰潭","盘锦","镇江","石嘴山","嘉兴","铜川","焦作","芜湖","潮州","萍乡","红河","黄石","辽源","辽阳","枣庄","毕节","许昌","阳泉","自贡","遂宁","景德镇","营口","日照","莆田"]//CommonUtils.findAllCity()
|
||||
for (String city : cities) {
|
||||
println city
|
||||
BOUNDS_COLL.remove(new BasicDBObject('city', city))
|
||||
@@ -70,13 +71,13 @@ class CityLatlngBox {
|
||||
int row = 0
|
||||
int inx = 0
|
||||
def buffer = []
|
||||
while(seedLatBD < cityBoundBD.getMaxLat()) {
|
||||
while(seedLatGD < cityBoundGD.getMaxLat()) {
|
||||
int col = 0
|
||||
Bounds boundsBD = null;
|
||||
Bounds boundsQQ = null;
|
||||
Bounds boundsGD = null;
|
||||
|
||||
while (seedLngBD < cityBoundBD.getMaxLng()) {
|
||||
while (seedLngGD < cityBoundGD.getMaxLng()) {
|
||||
double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // 右侧OFFSET处的经纬度
|
||||
double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // 上方OFFSET处的经纬度
|
||||
|
||||
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 小区建筑类型数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommbuildtypeBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
|
||||
def rows = createDatas(cityCodeDatas.get(city), dists)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
}
|
||||
|
||||
def createDatas(cityCode, dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCode, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
|
||||
println dist.city +" "+dist.region+" "+dist.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(cityCode, dist){
|
||||
def ret = []
|
||||
ret << cityCode
|
||||
ret << dist._id.toString()
|
||||
ret << CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
|
||||
}
|
||||
}
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 小区数据同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
|
||||
println city +" 小区量:"+dists.size()
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println city +" "+rets.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
allDatas << headers
|
||||
allDatas.addAll(rows)
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
if(!fangjiaDists.contains(ccbId)){
|
||||
def datas = ccbMaps.get(ccbId)
|
||||
datas[datas.size()-1] = 1
|
||||
rets << datas
|
||||
}
|
||||
}
|
||||
println "添加删除状态小区:"+rets.size()
|
||||
return rets
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
|
||||
//def rets = Collections.synchronizedList(new ArrayList());
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDataMap, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(def cityCodeDataMap, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
try{
|
||||
def data = distCodeMap.get(dist.region)
|
||||
if(!data){
|
||||
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret << cityCodeDataMap.get(dist.city)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist._id.toString()
|
||||
if(data){
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
def lat = dist.lat/10000000000d
|
||||
def lng = dist.lng/10000000000d
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "pa100"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
|
||||
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = null
|
||||
if(dist.address){
|
||||
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
}
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << 0
|
||||
}catch(Exception e){
|
||||
println dist.city + " "+dist.name
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
+543
@@ -0,0 +1,543 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 成交数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = 20210701//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataDealBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
println citys.unique().size()
|
||||
def rows = []
|
||||
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){
|
||||
// ppeDists = getPpeGTE15Datas(city)
|
||||
// }
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println city +" 成交共:"+rets.size()
|
||||
|
||||
|
||||
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
// def rets = []
|
||||
// rets << headers
|
||||
// rets.addAll(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(source == "youda"){
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
}
|
||||
}else{
|
||||
def ret = converData(trade, cityCodeDatas, distCodeMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
def data = distCodeMap.get(trade.fjRegion)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}
|
||||
if(!street){
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
}
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.dist_id.toString()
|
||||
ret << trade.master_name
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
|
||||
ret << ""//期数
|
||||
ret << trade.floor
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.avg_price
|
||||
ret << trade.total_price
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
def root = sourceMap.get(trade.source)
|
||||
if(root){
|
||||
if(CcbCommonUtils.rootMap.get(root)){
|
||||
ret << CcbCommonUtils.rootMap.get(root)
|
||||
}else{
|
||||
ret << "99"
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.total_price
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getHouseRoom(def room, def hall, def toilet){
|
||||
def s = ""
|
||||
if(room){
|
||||
s = s + room +"室"
|
||||
}
|
||||
if(hall){
|
||||
s = s + hall +"厅"
|
||||
}
|
||||
if(toilet){
|
||||
s = s + toilet +"卫"
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
def startDate = null;
|
||||
// if(citys.contains(city)){
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// }else{
|
||||
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// }
|
||||
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def price = statMap.get(dist.region+"@"+dist.name)
|
||||
def startPrice = null
|
||||
def endPrice = null
|
||||
// if(price){
|
||||
// if(dist.city=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else if(dist.city=="杭州"){
|
||||
// startPrice = (int)(price - price*0.15)
|
||||
// endPrice = (int)(price*0.15 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// }
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
|
||||
if(!trades && dist.address){
|
||||
def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)
|
||||
}
|
||||
|
||||
}
|
||||
if(trades){
|
||||
// def unitPrice= ppeDistsMap.get(dist._id.toString())
|
||||
// if(unitPrice){
|
||||
// for(def t : trades){
|
||||
// t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
|
||||
// }
|
||||
// }
|
||||
tradesList.addAll(trades)
|
||||
}
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return tradesList
|
||||
}
|
||||
|
||||
def getYouDaData(statMap){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
//def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
// q.put("premisesName" , "呼玛二村")
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def obj : dbcursor){
|
||||
|
||||
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
|
||||
rets << obj;
|
||||
// def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
|
||||
// if(price){
|
||||
// def startPrice = 0
|
||||
// def endPrice= 0
|
||||
// if(obj.fjCity=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// if(obj.price>=startPrice && obj.price<=endPrice){
|
||||
// rets << obj;
|
||||
// }else{
|
||||
// index++
|
||||
// println index + " "+obj
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def getPpeGTE15Datas(def city){
|
||||
def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " +
|
||||
"and cityname='"+city+"'";
|
||||
def map = [:]
|
||||
try{
|
||||
sqlInstance.eachRow(sql,{
|
||||
map.put(it.commid, it.unit_price)
|
||||
});
|
||||
}catch(Exception e){
|
||||
getPpeGTE15Datas(city)
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 区域数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDistrictBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataDistrictBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def datas = findDistrictCodeValue(city)
|
||||
if(datas){
|
||||
rows.addAll(datas)
|
||||
}
|
||||
println city+" "+ datas.size()
|
||||
}
|
||||
|
||||
def allDatas = createDatas(rows)
|
||||
|
||||
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
def allRows = []
|
||||
def index = 1
|
||||
rows.each { m ->
|
||||
def data = []
|
||||
data << m.dist_code
|
||||
data << m.dist_name
|
||||
data << m.dist_spell
|
||||
data << m.city_id
|
||||
data << m.dist_code
|
||||
data << index
|
||||
data << ""
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "FJW"
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
index++
|
||||
allRows << data
|
||||
}
|
||||
return allRows;
|
||||
}
|
||||
|
||||
def findDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
// q.put("district", new BasicDBObject('$ne',"市辖区"))
|
||||
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def lists = []
|
||||
for(def db : dbcursor){
|
||||
if(db.district_code){
|
||||
def m = [:]
|
||||
m.id = db._id.toString()
|
||||
m.dist_name = db.district
|
||||
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
|
||||
m.city_id = db.city_code
|
||||
m.dist_code = db.district_code
|
||||
lists << m
|
||||
}
|
||||
}
|
||||
return lists;
|
||||
}
|
||||
}
|
||||
+607
@@ -0,0 +1,607 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataForsaleBatch extends BaseBatch{
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def index = 0
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200101, 20210930);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def commId = rMap.get("_id").toString()
|
||||
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret && ret.size()<100){
|
||||
ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||
}
|
||||
if(ret){
|
||||
def areaData = distCodeMap.get(rMap.region)
|
||||
if(areaData){
|
||||
def regionCode = areaData.district_code
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = areaData.district
|
||||
m.region_code = regionCode
|
||||
m.district_id = commId
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
}
|
||||
}else{
|
||||
println city +" "+rMap.region+" 没有找到对应的区域"
|
||||
}
|
||||
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
|
||||
}
|
||||
|
||||
println city + " 房源总量:"+retMap.size()
|
||||
// def allDatas = createSellDatas(city, retMap.values());
|
||||
def allDatas = retMap.values()
|
||||
def headers = []
|
||||
if(index == 1){
|
||||
headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
}
|
||||
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
|
||||
println "txt文件生成 "
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createSellDatas(def city, def values){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(values.size())
|
||||
println city +"总房源量:"+values.size()
|
||||
|
||||
def datas = Collections.synchronizedList(new ArrayList());
|
||||
for(def i=0;i<values.size();i++){
|
||||
final def m = values[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
datas << converData(m)
|
||||
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return datas ;
|
||||
}
|
||||
|
||||
def converData(def m){
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
data << m.region
|
||||
data << m.district_id
|
||||
data << m.district_name
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.address==null?"":m.address==null
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << ""
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
data << m.hall==null?"":m.hall
|
||||
data << ""
|
||||
data << m.toilet==null?"":m.toilet
|
||||
data << m.direction_code==null?"":m.direction_code
|
||||
data << m.fitment_code==null?"":m.fitment_code
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.p_date==null?"":m.p_date
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.root
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.c_date==null?"":m.c_date
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
data << ""
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
// if(seawater.height && seawater.height<1){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
|
||||
println name +" "+houses.size()+" last:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public boolean isVilla(area, name, height, room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String csvHandlerStr(String str) {
|
||||
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
String tempDescription=str;
|
||||
//如果有逗号
|
||||
if(str.contains(",")||str.contains(",")){
|
||||
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if(str.contains("\"")){
|
||||
// tempDescription=str.replace("\"", "\"\"");
|
||||
tempDescription=str.replace("\"", "");
|
||||
}
|
||||
//在将逗号转义
|
||||
tempDescription="\""+tempDescription+"\"";
|
||||
}
|
||||
return tempDescription;
|
||||
}
|
||||
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
def getDistrictIds(def cityid){
|
||||
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
|
||||
|
||||
def distIds = new HashSet()
|
||||
sqlInstance.eachRow(sql,{it->
|
||||
distIds << it.commid
|
||||
});
|
||||
return distIds;
|
||||
}
|
||||
|
||||
}
|
||||
+293
@@ -0,0 +1,293 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import org.bson.types.ObjectId
|
||||
/**
|
||||
* 建行数据poi同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataPoiBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataPoiBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
|
||||
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
||||
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
|
||||
def rets = []
|
||||
|
||||
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows1)
|
||||
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
rets.addAll(rows1)
|
||||
|
||||
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
rets.addAll(rows2)
|
||||
//
|
||||
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows3)
|
||||
|
||||
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
rets.addAll(rows4)
|
||||
|
||||
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
rets.addAll(rows5)
|
||||
//
|
||||
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
// rets.addAll(rows6)
|
||||
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_POI_GJ.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
|
||||
// def headers1 = ["城市ID","城市","行政区ID","区域名称","经度","纬度","创建时间","来源公司"]
|
||||
// def regionCenterDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\行政区中心点经纬度.xlsx", 0,1)
|
||||
// def regionRets = createRegionCenters(regionCenterDatas)
|
||||
// FileUtils.batchCreateFile(regionRets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\region_center.txt", headers1, "utf-8", false);
|
||||
// println "行政区域中心txt 生成"
|
||||
|
||||
}
|
||||
|
||||
def createRegionCenters(def datas){
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def ret = []
|
||||
ret << d['A'].toInteger().toString()
|
||||
ret << d['B']
|
||||
ret << d['C'].toInteger().toString()
|
||||
ret << d['D']
|
||||
def loactionStr = d['E']
|
||||
if(loactionStr){
|
||||
def arrs = loactionStr.split(",")
|
||||
def loactionMap = GeoUtils.gcjToGps84(arrs[1].toDouble(), arrs[0].toDouble())
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
return rets;
|
||||
|
||||
}
|
||||
|
||||
def createDatas(citys, cityCodeDataMap, datas, type){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
def cityDistMap = CcbCommonUtils.findAllCityDistrictCodeValue()
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(citys.contains(d['D'])){
|
||||
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
println totalCount.get()+" "+ d['D']+" "+d['F']
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getRegionByLocation(lng, lat){
|
||||
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
|
||||
|
||||
if(jsonObject && jsonObject.info=="OK"){
|
||||
def compent = jsonObject.regeocode.addressComponent
|
||||
if(compent){
|
||||
def district = compent.district
|
||||
return district;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def addValue2(d, cityCodeDataMap, cityDistMap, type){
|
||||
def ret = []
|
||||
def citycode = cityCodeDataMap.get(d['E'])
|
||||
ret << citycode
|
||||
|
||||
def region =getRegionByLocation(d['H'], d['I'])
|
||||
def distCodeMap = cityDistMap.get(d['E'])
|
||||
if(region){
|
||||
region = RegionData.formatRegion(d['E'], region)
|
||||
}
|
||||
if(region){
|
||||
def distObj = distCodeMap.get(region)
|
||||
if(distObj){
|
||||
ret << distObj.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
println d['E']+" " + region+" 无对应的区域编码 "+d['F']
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(d['A']){
|
||||
ret << d['A']
|
||||
}else{
|
||||
ret << new ObjectId()
|
||||
|
||||
}
|
||||
|
||||
ret << d['G']
|
||||
ret << type
|
||||
|
||||
if(type=="地铁"){
|
||||
ret << "cz125"
|
||||
}else if(type=="公交"){
|
||||
ret << "cz123"
|
||||
}else if(type=="医院"){
|
||||
ret << "hp111"
|
||||
}else if(type == "银行"){
|
||||
ret << "se118"
|
||||
}else if(type =="丧葬服务"){
|
||||
ret << "ob101"
|
||||
}else if(type =="学校"){
|
||||
if(d['B'].contains("141202")){
|
||||
ret << "ed138"
|
||||
}else if(d['B'].contains("141203")){
|
||||
ret << "ed114"
|
||||
}
|
||||
}
|
||||
if(d['I']){
|
||||
def lat = d['I'].toDouble()
|
||||
def lng = d['H'].toDouble()
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << d['C']
|
||||
ret << d['F']
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
def addValue(d, cityCodeDataMap, cityDistMap, type){
|
||||
def ret = []
|
||||
def citycode = cityCodeDataMap.get(d['D'])
|
||||
ret << citycode
|
||||
def distCodeMap = cityDistMap.get(d['D'])
|
||||
def region =RegionData.formatRegion(d['D'], d['E'])
|
||||
|
||||
if(region){
|
||||
|
||||
def distObj = distCodeMap.get(region)
|
||||
if(distObj){
|
||||
ret << distObj.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << d['A']
|
||||
ret << d['F']
|
||||
ret << type
|
||||
|
||||
if(type=="地铁"){
|
||||
ret << "cz125"
|
||||
}else if(type=="公交"){
|
||||
ret << "cz123"
|
||||
}else if(type=="医院"){
|
||||
ret << "hp111"
|
||||
}else if(type == "银行"){
|
||||
ret << "se118"
|
||||
}else if(type =="丧葬服务"){
|
||||
ret << "ob101"
|
||||
}else if(type =="学校"){
|
||||
if(d['F'].contains("中学")){
|
||||
ret << "ed138"
|
||||
}else if(d['F'].contains("小学")){
|
||||
ret << "ed114"
|
||||
}
|
||||
}
|
||||
if(d['I']){
|
||||
def lat = d['I'].toDouble()
|
||||
def lng = d['H'].toDouble()
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
//
|
||||
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
|
||||
// public static def citys = ["武汉"]
|
||||
|
||||
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
|
||||
|
||||
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
public static DSCollection dataStoreConnection = null;
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
public static def findCityCodeValue(){
|
||||
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
|
||||
return cityCodeMap;
|
||||
}
|
||||
|
||||
public static def findAllDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
//q.put("district", new BasicDBObject('$ne',"市辖区"))
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
map.put(db.fangjia_district, db)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public static def findAllCityDistrictCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def distMap = map.get(db.fangjia_city)
|
||||
if(!distMap){
|
||||
distMap = [:]
|
||||
map.put(db.fangjia_city, distMap)
|
||||
}
|
||||
distMap.put(db.fangjia_district, db)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public static def findPersonPrice(city, date){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)));
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
public static def findGuidePrice(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(dist.name.endsWith("弄小区") || dist.name.endsWith("小区") ){
|
||||
def name = dist.name.replace("弄小区","弄").replace("小区","")
|
||||
if(!alias.contains(name)){
|
||||
alias << name
|
||||
}
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
// if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
d.build_type = dist.build_type
|
||||
d.build_structure = dist.build_structure
|
||||
d.house_roperty = dist.house_roperty
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
}
|
||||
// }
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("district_name", dist.address)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
d.build_type = dist.build_type
|
||||
d.build_structure = dist.build_structure
|
||||
d.house_roperty = dist.house_roperty
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
|
||||
public static def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
|
||||
if(count && avg_price){
|
||||
def retMap = [:]
|
||||
retMap.count = count
|
||||
retMap.avg_price = avg_price
|
||||
return retMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static DSCollection getDataStoreConnection(){
|
||||
if(dataStoreConnection != null) {
|
||||
return dataStoreConnection
|
||||
}
|
||||
DataStore ds = new DataStore("seaweed.db.fangjia.com", 27007)
|
||||
dataStoreConnection = ds.getCollection("fangjia.seaweed")
|
||||
return dataStoreConnection
|
||||
}
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","御海阳光")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def findDistrict(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
q.put("name", name);
|
||||
q.put("cat", cat)
|
||||
q.put("status", 0)
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def d : cursor){
|
||||
if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
return d;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public static def findCcbDistrict(cityid){
|
||||
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
|
||||
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where cityId=${cityid}";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DISTID
|
||||
data << r.COMMID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMNAME
|
||||
data << r.ADDRESS
|
||||
data << r.LATITUDE
|
||||
data << r.LONGITUDE
|
||||
data << r.LANDAREA
|
||||
data << r.BUILDAREA
|
||||
data << r.GREENINGRATE
|
||||
data << r.PLOTRATIO
|
||||
data << r.PARKINGCNT
|
||||
data << r.BUILDYEAR
|
||||
data << r.HOUSEUSE
|
||||
data << r.ZTS
|
||||
data << r.WORKDATE
|
||||
data << r.UPDATEFLAG
|
||||
data << r.UPDATETIME
|
||||
data << r.COMMTYPE
|
||||
data << r.COMMALIAS
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.IFNEW
|
||||
data << ""
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.XZJDB
|
||||
data << r.LJXZQHDM
|
||||
data << r.LJXZQH
|
||||
data << r.STATUS
|
||||
map.put(r.COMMID, data)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jiangnanblank.SFTP
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class CcbFtpUtils {
|
||||
|
||||
static def ftpServer = "192.168.0.45"
|
||||
static def ftpPort = 17495
|
||||
static def ftpUser = "ccb"
|
||||
static def ftpPassword = "0I1k8EAM+O8iA"
|
||||
static def ftpPath = "/home/ccb/output/"
|
||||
|
||||
static SFTP sftp = new SFTP();
|
||||
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
|
||||
|
||||
static def ftpUpload(outputPath_dist, month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
|
||||
}
|
||||
|
||||
static def reConnect(){
|
||||
if(sftpChannel == null){
|
||||
println "sftpChannel 为空,需重连....."
|
||||
if(sftp == null){
|
||||
sftp = new SFTP();
|
||||
}
|
||||
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
}else{
|
||||
println "sftpChannel 正常....."
|
||||
}
|
||||
return sftpChannel;
|
||||
}
|
||||
|
||||
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
|
||||
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
|
||||
|
||||
def data = new FileInputStream(new File(dataFileName))
|
||||
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 城市数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class CityBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new CityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def index = 1
|
||||
for(def city : citys){
|
||||
def data = CcbCommonUtils.findCityCodeValue()
|
||||
def map = [:]
|
||||
map.name = city
|
||||
if(data){
|
||||
map.city_code = data.city_code
|
||||
map.prov_code = data.province_code
|
||||
}
|
||||
def cityObj = findCityInfo(city)
|
||||
if(cityObj){
|
||||
map.city_spell = cityObj.spell
|
||||
map.city_level = cityObj.city_grade
|
||||
map.seq = index
|
||||
map.ifcapital = isCapital(cityObj.province, city)
|
||||
map.ifmunicipal = ifMunicipal(city)
|
||||
}
|
||||
index++
|
||||
rows << map
|
||||
}
|
||||
|
||||
def allDatas = createDatas(rows)
|
||||
def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
def allRows = []
|
||||
rows.each { m ->
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.name
|
||||
data << m.city_spell
|
||||
data << m.prov_code
|
||||
data << m.city_code
|
||||
data << m.city_level
|
||||
data << m.seq
|
||||
data << CcbCodeUtils.parseFlag(m.ifcapital)
|
||||
data << CcbCodeUtils.parseFlag(m.ifmunicipal)
|
||||
data << ""
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "fjw"
|
||||
allRows << data
|
||||
}
|
||||
return allRows;
|
||||
}
|
||||
|
||||
def findCityInfo(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("display", city)
|
||||
return CcbCommonUtils.SERVER_AREA_COLL.findOne(q)
|
||||
}
|
||||
|
||||
def isCapital(def prov, def city){
|
||||
def provCity = CcbCommonUtils.provCityMap.get(prov)
|
||||
if(provCity && provCity==city){
|
||||
return "是"
|
||||
}
|
||||
return "否"
|
||||
}
|
||||
|
||||
def ifMunicipal(def city){
|
||||
if(["上海", "北京", "天津", "重庆"].contains(city)){
|
||||
return "是"
|
||||
}
|
||||
return "否"
|
||||
}
|
||||
}
|
||||
+262
@@ -0,0 +1,262 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class DistrictDelete {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
// task.gzRun();
|
||||
// task.deleteDistrict()
|
||||
task.deleteCommPrice()
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"泉州","深圳", "重庆",
|
||||
def citys = ["广州"];//"广州",
|
||||
|
||||
def delIndex = 0
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
|
||||
|
||||
checkDist(city, dists)
|
||||
}
|
||||
}
|
||||
|
||||
def checkDist(city,dists){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd")
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def delIndex = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
println city +" 小区量:"+dists.size()
|
||||
|
||||
def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def dist = dists[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def count = findHouses(city, dist.region, "sell", 20190101,20211231, dist.name, dist.alias)
|
||||
def count = findHouseCount(city, dist.region, dist.name, dist.alias)
|
||||
def tradeCount = 0
|
||||
if(!count){
|
||||
tradeCount = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, 0, 300000)
|
||||
if(!tradeCount){
|
||||
def nameFlag = false
|
||||
for(def key : keys){
|
||||
if(dist.name.contains(key)){
|
||||
nameFlag = true
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(!nameFlag){
|
||||
if(dist.name.size()<=3 && (dist.name.contains("村") || dist.name.contains("楼") || dist.name.contains("居") )){
|
||||
nameFlag = true
|
||||
}
|
||||
}
|
||||
def fieldFlag = false
|
||||
if(dist.finish_building_date && dist.build_type){
|
||||
fieldFlag = true
|
||||
}
|
||||
|
||||
if(!tradeCount && !count && !fieldFlag){
|
||||
def rowCount = findCount(dist._id.toString())
|
||||
if(rowCount || nameFlag){
|
||||
delIndex.addAndGet(1)
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',q))
|
||||
println "[ ${delIndex.get()}/${totalCount.get()} ] "+city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
|
||||
}else{
|
||||
println city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
|
||||
}
|
||||
|
||||
rets << [dist._id.toString(),dist.city, dist.region, dist.name, dist.address]
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\多余小区_${city}.xlsx",rets, "无房源")
|
||||
}
|
||||
|
||||
|
||||
def findCount(distId){
|
||||
def sql = "select count(*) as count from fjw_community_artifi_check where commid='${distId}'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
static def findHouseCount(def city, def region, def name, def alias){
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and region = '"+region+"'")
|
||||
sqlBuffer.append(" and district_name in ('${aliasStr}')")
|
||||
sqlBuffer.append(" and (cat = 'rent' or cat='sell')")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def count = null
|
||||
while (rs.next()){
|
||||
count = rs.getInt("count")
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def gzRun(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\要删除小区 (5).xlsx",{d->
|
||||
if(d['G']){
|
||||
def id = d['G']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
index++
|
||||
println index
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
def id = d['G']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
deleteAllRefDistrict(id)
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteAllRefDistrict(id){
|
||||
def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"]
|
||||
for(def tableName : tables){
|
||||
deleteByCommid(tableName, id)
|
||||
}
|
||||
}
|
||||
|
||||
def deleteCommPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\均价删除.xlsx",{d->
|
||||
if(d['B']){
|
||||
def id = d['B']
|
||||
println id
|
||||
deleteByCommid("community_base_price_gj",id)
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteByCommid(tableName, commid){
|
||||
def sql = "delete from "+tableName+" where commid='${commid}'";
|
||||
sqlInstance.execute(sql)
|
||||
println "删除:"+tableName+" "+commid
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictGuidePriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictGuidePriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findGuidePrice(city)
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
}
|
||||
|
||||
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.match_region+"##"+ps.match_name)
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
if(areaData){
|
||||
ret << ps._id.toString()
|
||||
ret << dist._id.toString()
|
||||
ret << dist.name
|
||||
ret << cityCodeDatas.get(dist.city)
|
||||
ret << dist.city
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << ps.avg_price
|
||||
ret << ps.source_type
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}else{
|
||||
println ps.match_region+"##"+ps.match_name
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 估价系数数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class EvaluateRateBatch extends BaseBatch{
|
||||
|
||||
static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new EvaluateRateBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def allDatas = createDatas(findEvaluationRate())
|
||||
def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def createDatas(def datas){
|
||||
def allRows = []
|
||||
datas.each { m ->
|
||||
if(m.rank_remark!="开发商装修"){
|
||||
def data = []
|
||||
data << m.id.toString()
|
||||
data << m.rate_type
|
||||
data << m.rate
|
||||
data << m.rank
|
||||
data << "全国"
|
||||
data << converFitment(m.rank_remark)
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
data << "FJW"
|
||||
allRows << data
|
||||
}
|
||||
}
|
||||
return allRows
|
||||
}
|
||||
|
||||
private def converFitment(rank_remark){
|
||||
if(rank_remark){
|
||||
if(rank_remark.contains("简单装修")){
|
||||
return "简装"
|
||||
}else if(rank_remark.contains("精装修")){
|
||||
return "精装"
|
||||
}else if(rank_remark.contains("豪华装修")){
|
||||
return "豪装"
|
||||
}
|
||||
}
|
||||
return rank_remark
|
||||
}
|
||||
|
||||
private def findEvaluationRate(){
|
||||
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'";
|
||||
def rets = []
|
||||
sqlInstance.eachRow(sql,{
|
||||
rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated]
|
||||
});
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+819
@@ -0,0 +1,819 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class ExportOracleData {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportOracleData()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
// exportBigdataDistrictData()
|
||||
// exportGuidedPrice()
|
||||
// exportCommunityGuidedPrice()
|
||||
// exportBigdataDeal()
|
||||
exportThirdPartyPrice()
|
||||
|
||||
// exportBigdataCommunity()
|
||||
// exportBigdataForsale()
|
||||
|
||||
// exportAvmEvaluationGJNB()
|
||||
// exportAVMEvaluationItemGJNB()
|
||||
// exportCommunityBasePrice()
|
||||
|
||||
// exportCommunityBasePriceGj()
|
||||
// exportCityPrice()
|
||||
// exportRegionPrice()
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
|
||||
def exportEvaluationRateData(){
|
||||
def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.RATE_TYPE
|
||||
data << r.RATE
|
||||
data << r.RANK
|
||||
data << r.CITY
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataPoiData(){
|
||||
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where cityId in (350200,150100)";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DISTID
|
||||
data << r.ID
|
||||
data << r.POI_NAME
|
||||
data << r.TYPE
|
||||
data << r.TYPE_CODE
|
||||
data << r.LATITUDE
|
||||
data << r.LONGITUDE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "BIGDATA_POI_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "BIGDATA_POI_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityStaging(){
|
||||
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMID
|
||||
data << r.MASTER_COMMID
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "COMMUNITY_STAGING_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "COMMUNITY_STAGING_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBuileType(){
|
||||
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID in (350200,150100)";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDTYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDistrictData(){
|
||||
|
||||
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where cityId in (350200,150100)";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.DISTSPELL
|
||||
data << r.CITYID
|
||||
data << r.DISTCODE
|
||||
data << r.SEQ
|
||||
data << r.DISTTYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.LJXZQH
|
||||
data << r.LJXZQHDM
|
||||
data << r.XZQHDM
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "BIGDATA_DISTRICT_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "BIGDATA_DISTRICT_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportGuidedPrice(){
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate<TO_DATE('2021-09-01', 'yyyy-MM-dd') and cityname='上海'";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.DEALDATE
|
||||
data << r.AVG_PRICE
|
||||
data << r.GUIDING_PERSON
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "GUIDED_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityGuidedPrice(){
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.AVGPRICE
|
||||
data << r.SOURCETYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202107")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
|
||||
println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDeal(){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
|
||||
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
|
||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||
"from bigdata_deal_gj where cityId in (350200,150100) and dealdate>=to_date('2021-08-01','yyyy-mm-dd')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
data << r.BUILDTYPE
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.CJJE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "BIGDATA_DEAL_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportThirdPartyPrice(){
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" +
|
||||
"TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" +
|
||||
"from third_party_price_gj";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.ZHUGE_AVG_PRICE
|
||||
data << r.ANJUKE_AVG_PRICE
|
||||
data << r.FANG_AVG_PRICE
|
||||
data << r.LIANJIA_AVG_PRICE
|
||||
data << r.CITYHOUSE_AVG_PRICE
|
||||
data << r.MAITIAN_AVG_PRICE
|
||||
data << r.KE_AVG_PRICE
|
||||
data << r.TAOFANG_AVG_PRICE
|
||||
data << r.ZHONGYUAN_AVG_PRICE
|
||||
data << r.TBS_AVG_PRICE
|
||||
data << r.QFANG_AVG_PRICE
|
||||
data << r.FANGTAN_AVG_PRICE
|
||||
data << r.FANGDUODUO_AVG_PRICE
|
||||
data << r.WIWJ_AVG_PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
|
||||
// CcbFtpUtils.ftpUpload(fileName, "202107")
|
||||
// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
|
||||
// println "THIRD_PARTY_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommunity(){
|
||||
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
|
||||
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100)
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DISTID
|
||||
data << r.COMMID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMNAME
|
||||
data << r.ADDRESS
|
||||
data << r.LATITUDE
|
||||
data << r.LONGITUDE
|
||||
data << r.LANDAREA
|
||||
data << r.BUILDAREA
|
||||
data << r.GREENINGRATE
|
||||
data << r.PLOTRATIO
|
||||
data << r.PARKINGCNT
|
||||
data << r.BUILDYEAR
|
||||
data << r.HOUSEUSE
|
||||
data << r.ZTS
|
||||
data << r.WORKDATE
|
||||
data << r.UPDATEFLAG
|
||||
data << r.UPDATETIME
|
||||
data << r.COMMTYPE
|
||||
data << r.COMMALIAS
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.IFNEW
|
||||
data << ""
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.XZJDB
|
||||
data << r.LJXZQHDM
|
||||
data << r.LJXZQH
|
||||
data << r.STATUS
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......"
|
||||
// CcbFtpUtils.ftpUpload(fileName, "202109")
|
||||
// CcbFtpUtils.ftpUpload(dataCodeFileName, "202109")
|
||||
// println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataForsale(){
|
||||
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
|
||||
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
|
||||
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where PUBLISHTIME>=to_date('2021-09-01','yyyy-mm-dd')";//(cityId=350200 or cityId=150100) and
|
||||
def allRows = []
|
||||
def index = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
index = index+1
|
||||
println index
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.FORSALEID
|
||||
data << r.PROPCODE
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.UNITPRICE
|
||||
data << r.TOTALPRICE
|
||||
data << r.PAYTYPE
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORCNT
|
||||
data << r.BUILDTYPE
|
||||
data << r.HOUSESTRU
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.IFELEVATOR
|
||||
data << r.ELEVATORHOUSE
|
||||
data << r.PUBLISHTIME
|
||||
data << r.EXPIRETIME
|
||||
data << r.IFFIRST
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.PUBLISHER
|
||||
data << r.PHONE
|
||||
data << ""
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.ATTACH
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "BIGDATA_FORSALE_GJ 数据文件输出完成......"
|
||||
// CcbFtpUtils.ftpUpload(fileName, "202107")
|
||||
// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
|
||||
// println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportAvmEvaluationGJNB(){
|
||||
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
|
||||
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
|
||||
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where (cityId=350200 or cityId=150100) and year_month<202108 ";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.COMMUNITY_NUM
|
||||
data << r.HAS_PRICE_COMMUNITY_NUM
|
||||
data << r.PRICE_COVERAGE
|
||||
data << r.ARTIFICIAL_CALC_NUM
|
||||
data << r.ARTIFICIAL_CALC_COVERAGE
|
||||
data << r.SAMPLE_CALC_NUM
|
||||
data << r.SAMPLE_CALC_COVERAGE
|
||||
data << r.ANN_CALC_NUM
|
||||
data << r.ANN_CALC_COVERAGE
|
||||
data << r.TEST_NUM
|
||||
data << r.TEST_COMM_NUM
|
||||
data << r.TEST_COMM_COVERAGE
|
||||
data << r.PPE15_COVERAGE
|
||||
data << r.RATE_MEDIAN
|
||||
data << r.COEFFICIENT
|
||||
data << r.DEAL_TEST_CASE_NUM
|
||||
data << r.DEAL_TEST_COMM_NUM
|
||||
data << r.DEAL_TEST_COMM_COVERAGE
|
||||
data << r.DEAL_PPE15_COVERAGE
|
||||
data << r.DEAL_COEFFICIENT
|
||||
data << r.DEAL_RATE_MEDIAN
|
||||
data << r.THIRD_TEST_CASE_NUM
|
||||
data << r.THIRD_TEST_COMM_NUM
|
||||
data << r.THIRD_TEST_COMM_COVERAGE
|
||||
data << r.THIRD_PPE15_COVERAGE
|
||||
data << r.THIRD_COEFFICIENT
|
||||
data << r.THIRD_RATE_MEDIAN
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
|
||||
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
|
||||
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202107")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
|
||||
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportAVMEvaluationItemGJNB(){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
|
||||
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where (cityId=350200 or cityId=150100) and year_month>=202108";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.CONFIDENCE
|
||||
data << r.TARGET_AVG_PRICE
|
||||
data << r.DEALDATE
|
||||
data << r.UNIT_PRICE
|
||||
data << r.BUILDAREA
|
||||
data << r.FITMENT
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORCNT
|
||||
data << r.DATA_TYPE
|
||||
data << r.RATE
|
||||
data << r.PE
|
||||
data << r.CREATETIME
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where (cityId=350200 or cityId=150100) and year_month<202108";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202107")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePriceGj(){
|
||||
|
||||
}
|
||||
|
||||
def exportCityPrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where (cityId=350200 or cityId=150100) and year_month>=202108";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "CITY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "CITY_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportRegionPrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where (cityId=350200 or cityId=150100) and year_month>=202108";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportLastFile(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
|
||||
def dataCodeFileName = "/ccb/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok"
|
||||
}
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "${curMonth}")
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import org.bson.types.ObjectId
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 人工指导价同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GuiDingPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
|
||||
//添加搜房的参考价
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
// def datas = findFangDatas(districts)
|
||||
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
|
||||
// rows.addAll(fangRets)
|
||||
|
||||
//添加保亭文昌的指导价
|
||||
def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
|
||||
def cityDataMap = [:]
|
||||
for(def d : cityDatas){
|
||||
cityDataMap.put(d['A'], d['E'])
|
||||
}
|
||||
def personDatas = findPersonDatas(districts, cityDataMap);
|
||||
def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
|
||||
rows.addAll(personRets)
|
||||
|
||||
}
|
||||
|
||||
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
// def rets = []
|
||||
// rets << headers
|
||||
// rets.addAll(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
def findFangDatas(def districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println " 小区量:"+districts.size()
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findMatchThirdPrice(dist)
|
||||
if(obj && obj.avg_price){
|
||||
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return result
|
||||
}
|
||||
|
||||
def findPersonDatas(def districts, def dataMap){
|
||||
def result = []
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
def dist = districts[i];
|
||||
def price = dataMap.get(dist._id.toString())
|
||||
if (price) {
|
||||
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
|
||||
println dist.city + " " + dist.region + " " + dist.name + " " + price
|
||||
}
|
||||
}
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
def findMatchThirdPrice(dist){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", dist.city)
|
||||
q.put("match_region", dist.region)
|
||||
q.put("match_name", dist.name)
|
||||
q.put("s_date", sDate)
|
||||
q.put("source","搜房")
|
||||
def obj = dataProcessColl.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
if(areaData){
|
||||
ret << ps._id.toString()
|
||||
ret << dist._id.toString()
|
||||
ret << dist.name
|
||||
ret << cityCodeDatas.get(dist.city)
|
||||
ret << dist.city
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.guiding_value
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createDatas(cityCodeDatas, distCodeMap, datas){
|
||||
def rets = []
|
||||
if(datas) {
|
||||
for (def ps : datas) {
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(ps.region)
|
||||
if (areaData) {
|
||||
ret << ps.row_id
|
||||
ret << ps.dist_id
|
||||
ret << ps.name
|
||||
ret << cityCodeDatas.get(ps.city)
|
||||
ret << ps.city
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.avg_price
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
|
||||
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class LoadGuide {
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
static main(args) {
|
||||
def task = new LoadGuide()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
|
||||
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
|
||||
}
|
||||
|
||||
def savePrice(filePath){
|
||||
def datas = ExcelUtils.read(filePath,0, 1)
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.region = d['A']
|
||||
o.name = d['C']
|
||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
if(rets){
|
||||
println rets.size()
|
||||
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
|
||||
}
|
||||
}
|
||||
|
||||
def saveDongGuangPrice(def filePath){
|
||||
def datas = ExcelUtils.read(filePath,0, 0)
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.region = d['A']
|
||||
o.name = d['B']
|
||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.house_count = d['C'].toString().toDouble().toInteger()
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
if(rets){
|
||||
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
|
||||
}
|
||||
}
|
||||
|
||||
def saveChengduoPrice(def filePath){
|
||||
def datas = ExcelUtils.read(filePath,2, 0)
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.region = d['B']
|
||||
o.name = d['C']
|
||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
if(rets){
|
||||
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class LocationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new LocationUpdate();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度\\经纬度人工核实结果_0923.xlsx",{d->
|
||||
index++
|
||||
if(d['C']){
|
||||
|
||||
println index +" "+d['A']+" "+d['B']+" "+d['C']
|
||||
def region = RegionData.formatRegion(d['A'], d['B'])
|
||||
if(d['H']){
|
||||
def status = d['H'].toString().toDouble().toInteger()
|
||||
if(status==1){
|
||||
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
if(obj){
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble())
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
if(gmap){
|
||||
lng2 = (gmap.get("lng")*10000000000).longValue()
|
||||
lat2 = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
|
||||
BasicDBObject q=new BasicDBObject()
|
||||
q.put("m_date", new Date())
|
||||
q.put("lat2", lat2)
|
||||
q.put("lng2", lng2)
|
||||
q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue())
|
||||
q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
|
||||
|
||||
}else{
|
||||
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
|
||||
}
|
||||
}else{
|
||||
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
if(obj){
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
|
||||
}else{
|
||||
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
|
||||
}
|
||||
}
|
||||
|
||||
}else{
|
||||
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
if(obj){
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
|
||||
}else{
|
||||
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 分期数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class StagingDistrictBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new StagingDistrictBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
|
||||
def fenQiMap = getFenQiMap(dists);
|
||||
def rets = createCityData(dists, fenQiMap)
|
||||
rows.addAll(rets)
|
||||
println city +" 分期量:"+rets.size()
|
||||
}
|
||||
|
||||
|
||||
def results =[]
|
||||
def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
|
||||
results << headers
|
||||
results.addAll(rows)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
def createCityData(def dists, def fenQiMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
if(d.master_district_name && d.master_district_name!="master"){
|
||||
def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name)
|
||||
if(masterDist){
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << d._id.toString()
|
||||
ret << masterDist._id.toString()
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
// else{
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("master_district_name", null)
|
||||
// udata.put("m_date", new Date())
|
||||
// println d.city +" "+d.name+" "+d._id.toString()
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getFenQiMap(def dists){
|
||||
def fenQiMap = [:]
|
||||
for(def d : dists){
|
||||
// if(d.master_district_name){
|
||||
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
// }
|
||||
}
|
||||
return fenQiMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.*;
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
//def citys = CcbCommonUtils.citys;
|
||||
//def rows = []
|
||||
//
|
||||
//for(def city : citys) {
|
||||
// def data = CcbCommonUtils.findCityCodeValue()
|
||||
// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
|
||||
//
|
||||
// def count = 0
|
||||
// sqlInstance.execute(sql,{
|
||||
// count = it.count
|
||||
// });
|
||||
// println city +" "+count
|
||||
//}
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx"
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
//for(def d : datas){
|
||||
// dealMap.put(d['A'], d['B'])
|
||||
//}
|
||||
//
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// d['Q'] = dealMap.get(d['C'])
|
||||
// }
|
||||
//},2)
|
||||
|
||||
|
||||
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
|
||||
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
//for(def d : datas){
|
||||
// def map = [:]
|
||||
// map.put("land_area",d['D'])
|
||||
// map.put("build_area",d['F'])
|
||||
// map.put("greeningrate_area",d['H'])
|
||||
// map.put("parkingcnt_area",d['J'])
|
||||
// map.put("zts_area",d['L'])
|
||||
// map.put("buildyear_area",d['N'])
|
||||
// map.put("plotratio",d['P'])
|
||||
//
|
||||
// dealMap.put(d['A'], map)
|
||||
//}
|
||||
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
//小区量
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def city = d['B']
|
||||
// if (city) {
|
||||
// def cityid = cityCodeDatas.get(city)
|
||||
// def total = findCount(cityid, sqlInstance)
|
||||
// d['O'] = total
|
||||
// }
|
||||
//},0)
|
||||
|
||||
////小区字段覆盖率
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// def city = d['A']
|
||||
// if(city){
|
||||
// def cityid = cityCodeDatas.get(city)
|
||||
// def total = findCount(cityid, sqlInstance)
|
||||
// d['AA'] = total
|
||||
//
|
||||
// def a = findEstateType(cityid, sqlInstance)
|
||||
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
|
||||
//
|
||||
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
|
||||
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
|
||||
//
|
||||
// def c = findFiledCount(cityid, 'address', sqlInstance)
|
||||
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
|
||||
//
|
||||
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
|
||||
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
|
||||
//
|
||||
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
|
||||
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
|
||||
//
|
||||
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
|
||||
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
|
||||
//
|
||||
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
|
||||
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
|
||||
//
|
||||
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
|
||||
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
|
||||
//
|
||||
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
|
||||
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
|
||||
//
|
||||
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
|
||||
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
|
||||
// println d
|
||||
// }
|
||||
//},1)
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance)
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(d['A']){
|
||||
// d['U'] = dealMap.get(d['A'])
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx"
|
||||
def datas = ExcelUtils.read(fileName, 0, 1)
|
||||
def dealMap = [:]
|
||||
for(def d : datas){
|
||||
dealMap.put(d['A'], d['B'])
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d->
|
||||
if(d['A']){
|
||||
d['W'] = dealMap.get(d['A'])
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
|
||||
//def updateIndex = 0
|
||||
//def deleteIndex = 0
|
||||
//ExcelUtils.modify(fileName,{d->
|
||||
// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){
|
||||
// def dist = CommonDatautils.findDistrictById(d['D'])
|
||||
// def city = dist.city
|
||||
// def region = RegionData.formatRegion(city, d['Q'])
|
||||
// if(region && dist.region!=region){
|
||||
// println city +" new:"+region +" old:"+d['C']
|
||||
// def udata = new BasicDBObject()
|
||||
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
|
||||
// if(obj){
|
||||
// deleteIndex++
|
||||
// udata.put("status", 1)
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
|
||||
// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
|
||||
// }else{
|
||||
// updateIndex++
|
||||
// udata.put("region", region)
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
|
||||
// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
|
||||
// }
|
||||
// }else{
|
||||
// println d['D']+" 不存在"
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
def findDealCount(sqlInstance){
|
||||
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-08-01','yyyy-mm-dd') and dealdate<to_date('2021-09-01','yyyy-mm-dd') group by cityname";
|
||||
def count = 0
|
||||
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
map.put(r.cityname, r.count)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findEstateType(cityid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def findCount(cityid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def findFiledCount(cityid, field, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null";
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
|
||||
//def updateIndex = 0
|
||||
//def delIndex = 0
|
||||
//def cityMap = [:]
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(!d['R']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['D'])
|
||||
// if(obj){
|
||||
// def ranges = cityMap.get(obj.city)
|
||||
// if(!ranges){
|
||||
// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
|
||||
// ranges = []
|
||||
// for(def r : dbcursor){
|
||||
// ranges << r
|
||||
// }
|
||||
// cityMap.put(obj.city, ranges)
|
||||
// }
|
||||
//
|
||||
// if(ranges){
|
||||
// def lng = obj.lng/10000000000.0
|
||||
// def lat = obj.lat/10000000000.0
|
||||
//
|
||||
// def region = null
|
||||
// for(def r : ranges){
|
||||
// try{
|
||||
// def flag = isInArea(lng, lat, r.polyline)
|
||||
// if(flag){
|
||||
// def regionStr = r.region
|
||||
// region = RegionData.formatRegion(obj.city, regionStr)
|
||||
// break
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
// println e.getMessage()
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// if(region && region!=obj.region){
|
||||
// updateIndex++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
|
||||
// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
|
||||
// }else if(region && region==obj.region){
|
||||
// println "[same] "+obj.city +" "+obj.name
|
||||
// }else{
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
|
||||
// delIndex++
|
||||
// println delIndex +" [delete] "+obj.city +" "+obj.name
|
||||
// }
|
||||
// }else{
|
||||
// println obj.city +" 没有此城市边框信息"
|
||||
// }
|
||||
//
|
||||
// }else{
|
||||
// println d['D']+" 没找到该小区"
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//},0);
|
||||
|
||||
|
||||
|
||||
//def city = "杭州"
|
||||
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
|
||||
//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city))
|
||||
//def ranges = []
|
||||
//for(def d : dbcursor){
|
||||
// ranges << d
|
||||
//}
|
||||
//def index =0
|
||||
//def updateIndex = 0
|
||||
//def delIndex = 0
|
||||
//for(def dist : dists){
|
||||
//// if(dist.region=="江干"){
|
||||
// def lat = dist.lat/10000000000.0
|
||||
// def lng = dist.lng/10000000000.0
|
||||
//
|
||||
// def regionStr =null
|
||||
// def region = null
|
||||
// for(def r : ranges){
|
||||
// def flag = isInArea(lng, lat, r.polyline)
|
||||
// if(flag){
|
||||
// regionStr = r.region
|
||||
// region = RegionData.formatRegion(dist.city, regionStr)
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
|
||||
// if(regionStr && region!=dist.region){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
|
||||
// if(obj){
|
||||
// delIndex++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
|
||||
// println delIndex+" [del]"+region +" "+dist.name
|
||||
// }else{
|
||||
// updateIndex++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
|
||||
// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name
|
||||
// }
|
||||
// }else{
|
||||
// index++
|
||||
// println index+" [没找到]"+dist.name
|
||||
// }
|
||||
//// }
|
||||
//}
|
||||
|
||||
|
||||
def isInArea(lng, lat, points){
|
||||
List<Point> pointList = InsidePolygon.createPoint2(points);
|
||||
//Point point = new Point(lng, lat);
|
||||
//def flag = InsidePolygon.aboutZone(point, pointList)
|
||||
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
|
||||
return flag;
|
||||
}
|
||||
|
||||
def getStreetInfo(def city, def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent && addressComponent.city.contains(city)){
|
||||
township = addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
return township;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class TestRepeat {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new TestRepeat()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
}
|
||||
|
||||
|
||||
def checkUpdateCount(){
|
||||
def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat")
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(newHouses.size())
|
||||
println " ·¿Ô´Á¿£º"+newHouses.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<newHouses.size();i++){
|
||||
final def map = newHouses[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findHouse(map.houseId, map.commId)
|
||||
if(obj){
|
||||
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def findHouse(def houseId, def commId){
|
||||
def sql = "select * from bigdata_forsale_gj where commid='${commId}' and forsaleid='${houseId}'";
|
||||
def obj = null
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
obj = r
|
||||
});
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def readTxt(def file){
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(new File(file))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
def arrs = line.split("|@|")
|
||||
def map = [:]
|
||||
map.houseId = arrs[1]
|
||||
map.commId = arrs[6]
|
||||
map.pdate = arrs[10]
|
||||
rets << map
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 第三方均价数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202108//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202106
|
||||
def endDate = 202108
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThirdPartyPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||
for(def city : citys){
|
||||
index++
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
|
||||
rows.addAll(rets)
|
||||
println index+" "+city+" "+rets.size()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat"
|
||||
}
|
||||
|
||||
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def ret = []
|
||||
ret << third.id.toString()
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
ret << cityCodeDatas.get(d.city)
|
||||
ret << d.city
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
ret << sDate
|
||||
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
ret << third.zhugezhaofang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
ret << third.anjuke_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
ret << third.soufang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.lianjia_esf_price){
|
||||
ret << third.lianjia_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.cityhouse_esf_price){
|
||||
ret << third.cityhouse_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.maitian_esf_price){
|
||||
ret << third.maitian_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.beike_esf_price){
|
||||
ret << third.beike_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.get("365house_esf_price")){
|
||||
ret << third.get("365house_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.zhongyuan_esf_price){
|
||||
ret << third.zhongyuan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.tuboshi_esf_price){
|
||||
ret << third.tuboshi_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
ret << third.qfang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangtan_esf_price){
|
||||
ret << third.fangtan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangduoduo_esf_price){
|
||||
ret << third.fangduoduo_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
ret << third.get("5i5j_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
dataMap.put("id", db._id.toString())
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
dataMap.put("soufang_esf_price", db.soufang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
dataMap.put("qfang_esf_price", db.qfang_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
def resultMap = [:]
|
||||
map.each {key, value->
|
||||
if(value && value.size()>1){
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
|
||||
def price = statMap.get(key)
|
||||
if(price){
|
||||
def priceMap = [:]
|
||||
def startPrice = (int)(price - price*0.3)
|
||||
def endPrice = (int)(price*0.3 + price)
|
||||
value.each{pk,pv->
|
||||
if(pk.contains("price")){
|
||||
if(pv>=startPrice && pv<=endPrice){
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}else{
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}
|
||||
if(priceMap.size()>1){
|
||||
resultMap.put(key, priceMap)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
resultMap.put(key, value)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return resultMap
|
||||
}
|
||||
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+652
@@ -0,0 +1,652 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/***
|
||||
* 建行成交抓取统计
|
||||
*/
|
||||
class TradeStatLog {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
task.exportTrade()
|
||||
// task.deleteDistrict()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-06-01"
|
||||
def date = "2021-07-01"
|
||||
def endDate = "2021-08-01"
|
||||
def newDate = "2021-09-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
def nearMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
|
||||
|
||||
def ret = []
|
||||
ret << city
|
||||
|
||||
if(city == "上海"){
|
||||
def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
def nearMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
|
||||
def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger()
|
||||
def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger()
|
||||
def c = nearMonthCount.toInteger() + nearMonthYoudaCount.toInteger()
|
||||
ret << a
|
||||
ret << b
|
||||
ret << c
|
||||
def d = (c-b)*100/a
|
||||
def pre = DataHandleUtils.getDoubleRound(2, d)
|
||||
ret << pre
|
||||
}else{
|
||||
ret << lastMonthCount
|
||||
ret << curMonthCount
|
||||
ret << nearMonthCount
|
||||
if(curMonthCount){
|
||||
def a = (nearMonthCount-curMonthCount)*100/curMonthCount
|
||||
def pre = DataHandleUtils.getDoubleRound(2, a)
|
||||
ret << pre
|
||||
}else{
|
||||
ret << null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for(def source : sources){
|
||||
def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
def sourceNearMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
|
||||
ret << sourceLastMonthCount
|
||||
ret << sourceCurMonthCount
|
||||
ret << sourceNearMonthCount
|
||||
}
|
||||
rets << ret
|
||||
println ret
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
def thirdPartyStat(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"]
|
||||
for(def city : citys){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat","lng"]);
|
||||
def m = getThirdPartyPricePrecent(city)
|
||||
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
|
||||
println rets.get(rets.size()-1)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计0927.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesCount(city, source, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
if(source){
|
||||
q.put("source", source)
|
||||
}
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def getYouDaData(startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lt', endDate));
|
||||
def count = youda_esfhouse_coll.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
def getThirdPartyPricePrecent(def city){
|
||||
def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108";
|
||||
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
def count3 = 0
|
||||
def count4 = 0
|
||||
def count5 = 0
|
||||
def count6 = 0
|
||||
def count7 = 0
|
||||
def count8 = 0
|
||||
def count9 = 0
|
||||
def count10 = 0
|
||||
def count11 = 0
|
||||
def count12 = 0
|
||||
def count13 = 0
|
||||
def count14 = 0
|
||||
def total = 0
|
||||
sqlInstance.eachRow(sql,{third->
|
||||
total++
|
||||
def ret = []
|
||||
if(third.zhuge_avg_price){
|
||||
ret << third.zhuge_avg_price
|
||||
}
|
||||
if(third.anjuke_avg_price){
|
||||
ret << third.anjuke_avg_price
|
||||
}
|
||||
if(third.fang_avg_price){
|
||||
ret << third.fang_avg_price
|
||||
}
|
||||
|
||||
if(third.lianjia_avg_price){
|
||||
ret << third.lianjia_avg_price
|
||||
}
|
||||
|
||||
if(third.cityhouse_avg_price){
|
||||
ret << third.cityhouse_avg_price
|
||||
}
|
||||
|
||||
if(third.maitian_avg_price){
|
||||
ret << third.maitian_avg_price
|
||||
}
|
||||
|
||||
if(third.ke_avg_price){
|
||||
ret << third.ke_avg_price
|
||||
}
|
||||
|
||||
if(third.taofang_avg_price){
|
||||
ret << third.taofang_avg_price
|
||||
}
|
||||
|
||||
if(third.zhongyuan_avg_price){
|
||||
ret << third.zhongyuan_avg_price
|
||||
}
|
||||
|
||||
if(third.tbs_avg_price){
|
||||
ret << third.tbs_avg_price
|
||||
}
|
||||
if(third.qfang_avg_price){
|
||||
ret << third.qfang_avg_price
|
||||
}
|
||||
|
||||
if(third.fangtan_avg_price){
|
||||
ret << third.fangtan_avg_price
|
||||
}
|
||||
|
||||
if(third.fangduoduo_avg_price){
|
||||
ret << third.fangduoduo_avg_price
|
||||
}
|
||||
if(third.wiwj_avg_price){
|
||||
ret << third.wiwj_avg_price
|
||||
}
|
||||
|
||||
if(ret.size()==0){
|
||||
count0++
|
||||
}
|
||||
if(ret.size()==1){
|
||||
count1++
|
||||
}
|
||||
if(ret.size()==2){
|
||||
count2++
|
||||
}
|
||||
if(ret.size()==3){
|
||||
count3++
|
||||
}
|
||||
if(ret.size()==4){
|
||||
count4++
|
||||
}
|
||||
if(ret.size()==5){
|
||||
count5++
|
||||
}
|
||||
|
||||
if(ret.size()==6){
|
||||
count6++
|
||||
}
|
||||
if(ret.size()==7){
|
||||
count7++
|
||||
}
|
||||
if(ret.size()==8){
|
||||
count8++
|
||||
}
|
||||
if(ret.size()==9){
|
||||
count9++
|
||||
}
|
||||
if(ret.size()==10){
|
||||
count10++
|
||||
}
|
||||
if(ret.size()==11){
|
||||
count11++
|
||||
}
|
||||
if(ret.size()==12){
|
||||
count12++
|
||||
}
|
||||
if(ret.size()==13){
|
||||
count13++
|
||||
}
|
||||
if(ret.size()==14){
|
||||
count14++
|
||||
}
|
||||
});
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
def pre3 = 0
|
||||
def pre4 = 0
|
||||
def pre5 = 0
|
||||
def pre6 = 0
|
||||
def pre7 = 0
|
||||
def pre8 = 0
|
||||
def pre9 = 0
|
||||
def pre10 = 0
|
||||
def pre11 = 0
|
||||
def pre12 = 0
|
||||
def pre13 = 0
|
||||
def pre14 = 0
|
||||
if(total){
|
||||
pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total)
|
||||
pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total)
|
||||
pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total)
|
||||
pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total)
|
||||
pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total)
|
||||
pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total)
|
||||
pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total)
|
||||
pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total)
|
||||
pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total)
|
||||
pre10 = DataHandleUtils.getDoubleRound(2, count10*100.0/total)
|
||||
pre11 = DataHandleUtils.getDoubleRound(2, count11*100.0/total)
|
||||
pre12 = DataHandleUtils.getDoubleRound(2, count12*100.0/total)
|
||||
pre13 = DataHandleUtils.getDoubleRound(2, count13*100.0/total)
|
||||
pre14 = DataHandleUtils.getDoubleRound(2, count14*100.0/total)
|
||||
}
|
||||
def map = [:]
|
||||
map.total = total
|
||||
map.count1 = pre1
|
||||
map.count2 = pre2
|
||||
map.count3 = pre3
|
||||
map.count4 = pre4
|
||||
map.count5 = pre5
|
||||
map.count6 = pre6
|
||||
map.count7 = pre7
|
||||
map.count8 = pre8
|
||||
map.count9 = pre9
|
||||
map.count10 = pre10
|
||||
map.count11 = pre11
|
||||
map.count12 = pre12
|
||||
map.count13 = pre13
|
||||
map.count14 = pre14
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def averageStat(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
|
||||
|
||||
def map = [:]
|
||||
for(def city : citys){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
|
||||
def rets = findThirdPriceByDist(districts)
|
||||
map.put(city, rets.size())
|
||||
map.put(city+"-count", districts.size())
|
||||
println districts.size()+" "+ rets.size()
|
||||
}
|
||||
println map
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\估值测试数据覆盖率占比.xlsx",{d->
|
||||
def city = d['A']
|
||||
def count = map.get(city)
|
||||
def distCount = map.get(city+"-count")
|
||||
d['H'] = count
|
||||
d['J'] = distCount
|
||||
|
||||
def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount)
|
||||
d['I'] = pre
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findThirdPriceByDist(districts){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def count = findAverageStat(dist)
|
||||
if(count){
|
||||
rets << dist
|
||||
println totalCount.get() +"/${districts.size()} "+dist.region+" "+dist.name
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findAverageStat(dist){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("name", dist.name)
|
||||
q.put("category","district")
|
||||
q.put("s_date", new BasicDBObject('$gte', 202105).append('$lt',202108))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def flag = 0
|
||||
for(def d : dbcursor){
|
||||
if(!flag){
|
||||
if(d.zhugezhaofang_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.anjuke_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.get("58tc_esf_price")){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.soufang_esf_price){
|
||||
flag = 1
|
||||
}
|
||||
if(d.lianjia_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.cityhouse_esf_price){
|
||||
continue
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.maitian_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.beike_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.get("365house_esf_price")){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.zhongyuan_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.tuboshi_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.qfang_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
|
||||
if(d.fangtan_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.fangduoduo_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.get("5i5j_esf_price")){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
def exportTrade(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
|
||||
// def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-07-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-08-01","yyyy-MM-dd"))
|
||||
// def trades = getYouDaData()
|
||||
|
||||
//def statMap = getDistrictStatByCity(city, weekend)
|
||||
def nameMap = getAllDistrict(city)
|
||||
|
||||
|
||||
def rMap = findDealIds(city);
|
||||
def ids = rMap.ids
|
||||
def details = rMap.detail
|
||||
|
||||
|
||||
for(def t : trades){
|
||||
if(!ids.contains(t._id.toString())){
|
||||
|
||||
def distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name)
|
||||
if(!distId && (t.district_name.toString().endsWith("弄") || t.district_name.toString().endsWith("号"))){
|
||||
distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name+"小区")
|
||||
}
|
||||
if(!distId){
|
||||
def flag = t.district_name.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
def datas = CommonDatautils.findDistrictByAddress2(t.city, t.region, t.district_name, "district")
|
||||
if(datas && datas.size()==1){
|
||||
distId = datas.get(0)._id.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def reason = ""
|
||||
if(distId){
|
||||
def dist = CommonDatautils.findDistrictById(distId)
|
||||
// def price = statMap.get(dist.region+"@"+dist.name)
|
||||
// def startPrice = null
|
||||
// def endPrice = null
|
||||
// if(price){
|
||||
// if(dist.city=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else if(dist.city=="杭州"){
|
||||
// startPrice = (int)(price - price*0.15)
|
||||
// endPrice = (int)(price*0.15 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// }
|
||||
// if(t.avg_price<startPrice || t.avg_price>endPrice){
|
||||
// reason = "价格过滤"
|
||||
// }
|
||||
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
|
||||
reason = "小区过滤"
|
||||
}
|
||||
|
||||
if(!reason){
|
||||
def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
|
||||
if(details.contains(key)){
|
||||
reason = "数据重复"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
|
||||
//rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type]
|
||||
rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason]
|
||||
}
|
||||
}
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
def id = d['G']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
deleteAllRefDistrict(id)
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteAllRefDistrict(id){
|
||||
def tables = ["bigdata_community_gj"]
|
||||
for(def tableName : tables){
|
||||
deleteByCommid(tableName, id)
|
||||
}
|
||||
}
|
||||
|
||||
def deleteByCommid(tableName, commid){
|
||||
def sql = "delete from "+tableName+" where commid='${commid}'";
|
||||
sqlInstance.execute(sql)
|
||||
println "删除:"+tableName+" "+commid
|
||||
}
|
||||
|
||||
def getYouDaData() {
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd")
|
||||
def q = new BasicDBObject()
|
||||
// q.put("premisesName","共富一村")
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate)
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
def trades = []
|
||||
for(def db : dbcursor){
|
||||
def map = [:]
|
||||
map._id=db._id.toString()
|
||||
map.city = "上海"
|
||||
map.region = db.region
|
||||
if(db.districtName && db.districtName!="null"){
|
||||
map.district_name=db.districtName
|
||||
}else if(db.address){
|
||||
map.district_name=db.address
|
||||
}
|
||||
|
||||
map.area = db.area
|
||||
map.trade_date = db.contractTime
|
||||
map.avg_price= db.price
|
||||
map.estate_type = db.houseType
|
||||
trades << map
|
||||
}
|
||||
return trades
|
||||
}
|
||||
|
||||
def findDealIds(city){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate<to_date('2021-08-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def ids = []
|
||||
def detailInfos = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.dealid)){
|
||||
ids << r.dealid
|
||||
|
||||
def region = RegionData.formatRegion(r.cityname, r.distname)
|
||||
def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd")
|
||||
if(!detailInfos.contains(key)){
|
||||
detailInfos << key
|
||||
}
|
||||
}
|
||||
});
|
||||
def rMap = [:]
|
||||
rMap.ids = ids
|
||||
rMap.detail = detailInfos
|
||||
return rMap
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesDetail(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("district_name","百合花园")
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def getAllDistrict(city){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
|
||||
def map = [:]
|
||||
for(def d : dists){
|
||||
map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString())
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
for(def name : alias){
|
||||
map.put(d.city+"##"+d.region+"##"+name, d._id.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
+392
@@ -0,0 +1,392 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class UpdateHouseRegion {
|
||||
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new UpdateHouseRegion()
|
||||
// task.updateSeawaterRegion()
|
||||
// task.updateDealRegion()
|
||||
// task.updateThirdPartyRegion()
|
||||
task.updateNoMatchDealRegion()
|
||||
// task.addAlias()
|
||||
// task.addFenQi()
|
||||
}
|
||||
|
||||
def addFenQi(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
|
||||
addFenQiAlias(dists)
|
||||
println city +" 处理完成"
|
||||
}
|
||||
}
|
||||
|
||||
def addFenQiAlias(dists){
|
||||
for(def dist : dists){
|
||||
def s1 = DataUtil.match(dist.name, Pattern.compile("[一|二|三|四|五|六|七|八|九|十|十一|十二|十三|十四|十五|十六|十七|十八|\\w+](期|区)\$"))
|
||||
if(s1){
|
||||
def newName = dist.name.replace("Ⅰ","1").replace("Ⅱ","2").replace("A区","a区").replace("B区","b区").replace("C区","c区").replace("D区","d区").replace("E区","e区").replace("F区","f区").replace("一期","1期").replace("二期","2期").replace("三期","3期").replace("四期","4期").replace("五期","5期").replace("六期","6期").replace("七期","7期").replace("八期","8期").replace("九期","9期").replace("十期","10期").replace("十一期","11期").replace("十二期","12期")
|
||||
if(newName!=dist.name){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+alias
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addAlias(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
|
||||
addDistricts(dists)
|
||||
println city +" 处理完成"
|
||||
}
|
||||
}
|
||||
|
||||
def addDistricts(dists){
|
||||
for(def dist : dists){
|
||||
def alias = dist.alias
|
||||
if(alias){
|
||||
def oldCount = alias.size()
|
||||
def newAlias = []
|
||||
for(def aliasName : alias){
|
||||
if(aliasName.contains("·") || aliasName.contains(".")){
|
||||
def newName = aliasName.replace("·","").replace(".","")
|
||||
newAlias << aliasName
|
||||
if(!newAlias.contains(newName)){
|
||||
if(newName!=dist.name){
|
||||
newAlias << newName
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(!newAlias.contains(aliasName)) {
|
||||
newAlias << aliasName
|
||||
}
|
||||
}
|
||||
}
|
||||
def newCount = newAlias.size()
|
||||
if(oldCount && alias!=newAlias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", newAlias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+newAlias
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateThirdPartyRegion(){
|
||||
def city = "东莞"
|
||||
def rets = findThirdPartyPrice(city)
|
||||
batchThirdPartyUpdate(city, rets);
|
||||
|
||||
}
|
||||
def findThirdPartyPrice(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date",new BasicDBObject('$gte', 202101))
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findDealData(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01")));
|
||||
def cursor = DEAL_PRICE_COLL.find(q).batchSize(2000)
|
||||
def datas = []
|
||||
def index = 0
|
||||
for(def obj : cursor){
|
||||
index++
|
||||
datas << obj
|
||||
println index
|
||||
}
|
||||
return datas
|
||||
}
|
||||
|
||||
def updateDealRegion(){
|
||||
def city = "东莞"
|
||||
def datas = findDealData(city)
|
||||
batchDealRegion(city, datas)
|
||||
}
|
||||
|
||||
def batchDealRegion(def city, def datas){
|
||||
def updateCount = new AtomicLong()
|
||||
def curCount = new AtomicLong()
|
||||
def totalCount = datas.size();
|
||||
final CountDownLatch latch = new CountDownLatch(totalCount)
|
||||
println city +" 成交总量:"+totalCount
|
||||
for(def i=0;i<totalCount;i++) {
|
||||
final def data = datas[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
curCount.addAndGet(1)
|
||||
try {
|
||||
def region = data.region
|
||||
def name = data.district_name
|
||||
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||
if (!rets) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||
}
|
||||
|
||||
if (rets.size() == 1) {
|
||||
def dist = rets[0]
|
||||
if (region != dist.region) {
|
||||
updateCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", dist.region)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
|
||||
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def updateNoMatchDealRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d->
|
||||
if(!d['J']){
|
||||
def city = d['B']
|
||||
def region = d['C']
|
||||
def name = d['D']
|
||||
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||
if (!rets) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||
}
|
||||
|
||||
if (rets.size() == 1) {
|
||||
def dist = rets[0]
|
||||
if (region != dist.region) {
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", dist.region)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
d['J'] = "原数据区域修改("+dist.region+")"
|
||||
println d['A']+" "+region +" "+dist.region
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateSeawaterRegion(){
|
||||
def city = "东莞"
|
||||
def rets = findHouseSource(city, 20190701, 20211001)
|
||||
batchUpdate(city, rets);
|
||||
}
|
||||
|
||||
|
||||
def batchThirdPartyUpdate(def city, def thirdDatas){
|
||||
def updateCount = new AtomicLong()
|
||||
def curCount = new AtomicLong()
|
||||
def totalCount = thirdDatas.size();
|
||||
final CountDownLatch latch = new CountDownLatch(totalCount)
|
||||
println city +" 第三方总量:"+totalCount
|
||||
for(def i=0;i<totalCount;i++) {
|
||||
final def data = thirdDatas[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
curCount.addAndGet(1)
|
||||
try {
|
||||
def region = data.region
|
||||
def name = data.name
|
||||
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||
if (!rets) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||
}
|
||||
|
||||
if (rets.size() == 1) {
|
||||
def dist = rets[0]
|
||||
if (region != dist.region) {
|
||||
updateCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", dist.region)
|
||||
thirdPartyColl.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
|
||||
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def findCount(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate)
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
|
||||
while (rs.next()) {
|
||||
def count = rs.getInt("count")
|
||||
println count
|
||||
}
|
||||
}
|
||||
|
||||
def batchUpdate(def city, def houses){
|
||||
def updateCount = new AtomicLong()
|
||||
def curCount = new AtomicLong()
|
||||
def totalCount = houses.size();
|
||||
final CountDownLatch latch = new CountDownLatch(totalCount)
|
||||
println city +" 房源总量:"+totalCount
|
||||
for(def i=0;i<totalCount;i++) {
|
||||
final def house = houses[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
curCount.addAndGet(1)
|
||||
try {
|
||||
def region = house.region
|
||||
def name = house.name
|
||||
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||
if (!rets) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||
}
|
||||
|
||||
if (rets.size() == 1) {
|
||||
def dist = rets[0]
|
||||
if (region != dist.region) {
|
||||
updateCount.addAndGet(1)
|
||||
updateHouseRegion2(house.url, dist.region);
|
||||
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def findHouseSource(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM \"seawater-2019\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
|
||||
def houses = []
|
||||
while (rs.next()){
|
||||
houses << ["region":rs.getString("region"), "name":rs.getString("district_name"),"url":rs.getString("url")]
|
||||
}
|
||||
return houses;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def updateHouseRegion2(def url, def region){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.206:9200/seawater-2019/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.region='${region}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"url": "${url}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
// println(response)
|
||||
}
|
||||
|
||||
|
||||
def updateHouseRegion(def url, def region){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("update seawater-2021")
|
||||
sqlBuffer.append(" set region = '"+region+"'")
|
||||
sqlBuffer.append(" where url='"+url+"'")
|
||||
def result = dataSource.connection.createStatement().executeUpdate(sqlBuffer.toString())
|
||||
println region +" "+result
|
||||
}
|
||||
|
||||
def getAllDistricts(def city){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"])
|
||||
def map = [:]
|
||||
for(def d : dists){
|
||||
|
||||
map.put()
|
||||
}
|
||||
}
|
||||
}
|
||||
+70
-4
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","东寓新村")
|
||||
// q.put("name","金辉伯爵山")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -536,8 +536,8 @@ class CommonDatautils {
|
||||
if(name){
|
||||
alias << name;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("region", region);
|
||||
@@ -698,6 +698,28 @@ class CommonDatautils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictByQuery(def q, def cat, def fields){
|
||||
|
||||
@@ -758,6 +780,9 @@ class CommonDatautils {
|
||||
keys.put("address", 1);
|
||||
keys.put("build_structure", 1);
|
||||
keys.put("out_pics", 1);
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("name_cat", 1);
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
@@ -865,6 +890,25 @@ class CommonDatautils {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static def findDistrictsByName(String city, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("cat", cat);
|
||||
q.put("name", name);
|
||||
q.put("visible", 0);
|
||||
|
||||
def rets = []
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
rets << obj;
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def getSimpleDistrictMap(def cat, def city){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible", 0);
|
||||
@@ -1187,6 +1231,28 @@ class CommonDatautils {
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrictByAddress2(def city, def region, def address, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('visible',0)
|
||||
q.put("address", address);
|
||||
|
||||
|
||||
def dsCursor = getDataStoreConnection().find(q);
|
||||
def ret = [];
|
||||
for(def obj : dsCursor){
|
||||
ret << obj;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public static def getCityRegions(city){
|
||||
@@ -1292,7 +1358,7 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
public static def getAllDictionary(){
|
||||
def dbCursor = DICT_COLL.find(new BasicDBObject())
|
||||
def dbCursor = DICT_COLL.find(new BasicDBObject()).batchSize(5000)
|
||||
def rets = []
|
||||
def filters = [ "estate","hfword","main","metro","station"]
|
||||
for(def obj : dbCursor){
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202106.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
@@ -135,7 +135,7 @@ rets << ["9
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
|
||||
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
|
||||
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
|
||||
|
||||
println s
|
||||
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
|
||||
|
||||
+171
-65
@@ -1,11 +1,16 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
@@ -63,7 +68,7 @@ class AddThirdPriceDistrict {
|
||||
def resAddrMap = [:]
|
||||
def filterEstateTypeMap = [:]
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -74,20 +79,84 @@ class AddThirdPriceDistrict {
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceDistrict();
|
||||
// task.run()
|
||||
task.run()
|
||||
|
||||
def mapName = "锦华星光苑3,4幢"
|
||||
// def mapName = "锦华星光苑3,4幢"
|
||||
//// println task.pattern("港新花苑A1区");
|
||||
println mapName.replaceAll("\\w+-?\\w+(幢)", "")
|
||||
println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
|
||||
// println mapName.replaceAll("\\w+-?\\w+(幢)", "")
|
||||
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
|
||||
}
|
||||
def run(){
|
||||
// exportDatas();
|
||||
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
// addInfo();
|
||||
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
|
||||
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
|
||||
addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
// getDataType("D:\\数据\\小区\\南昌第三方均价.xlsx");
|
||||
// addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx");
|
||||
// addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
// addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
// dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
// matchDistDatas("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
|
||||
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
}
|
||||
|
||||
def dataFilter(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def name = row['C']
|
||||
if(name){
|
||||
def flag = NameProcess.filterName(name)
|
||||
if(!flag){
|
||||
row['T'] = 1
|
||||
}else{
|
||||
if(name.toString().length()<3){
|
||||
if(row['N'] && row['N']>=10 && !row['J']){
|
||||
row['T'] = 25
|
||||
}else{
|
||||
row['T'] = 18
|
||||
}
|
||||
|
||||
}else{
|
||||
def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
|
||||
if(s1){
|
||||
row['T'] = 19
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(!name){
|
||||
row['T'] = 1
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
println "3.名称过滤完成"
|
||||
}
|
||||
|
||||
def addNewMapInfo(def path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['I']){
|
||||
|
||||
def gdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "高德")
|
||||
if(gdData){
|
||||
row['F'] = gdData.district
|
||||
row['G'] = gdData.name
|
||||
row['H'] = gdData.address
|
||||
row['I'] = gdData.type
|
||||
row['J'] = gdData.score
|
||||
row['K'] = gdData.lng
|
||||
row['L'] = gdData.lat
|
||||
}
|
||||
|
||||
def bdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "百度")
|
||||
if(bdData){
|
||||
row['M'] = bdData.district
|
||||
row['N'] = bdData.name
|
||||
row['O'] = bdData.address
|
||||
row['P'] = bdData.type
|
||||
row['Q'] = bdData.score
|
||||
row['R'] = bdData.lng
|
||||
row['S'] = bdData.lat
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def addCrawlMapInfo( def path){
|
||||
@@ -105,27 +174,27 @@ class AddThirdPriceDistrict {
|
||||
row['K'] = data.lat
|
||||
}
|
||||
|
||||
def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
|
||||
def bdRegion =null
|
||||
def bdName = null
|
||||
def bdAddress = null
|
||||
def bdLng = null
|
||||
def bdLat = null
|
||||
def bdEstateType = null
|
||||
if(baiduRets){
|
||||
bdRegion = baiduRets.region
|
||||
bdName = baiduRets.name
|
||||
bdAddress = baiduRets.address
|
||||
bdLng = baiduRets.lng
|
||||
bdLat = baiduRets.lat
|
||||
bdEstateType = baiduRets.estateType
|
||||
}
|
||||
row['L'] = bdRegion
|
||||
row['M'] = bdName
|
||||
row['N'] = bdAddress
|
||||
row['O'] = bdEstateType
|
||||
row['P'] = bdLng
|
||||
row['Q'] = bdLat
|
||||
// def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
|
||||
// def bdRegion =null
|
||||
// def bdName = null
|
||||
// def bdAddress = null
|
||||
// def bdLng = null
|
||||
// def bdLat = null
|
||||
// def bdEstateType = null
|
||||
// if(baiduRets){
|
||||
// bdRegion = baiduRets.region
|
||||
// bdName = baiduRets.name
|
||||
// bdAddress = baiduRets.address
|
||||
// bdLng = baiduRets.lng
|
||||
// bdLat = baiduRets.lat
|
||||
// bdEstateType = baiduRets.estateType
|
||||
// }
|
||||
// row['L'] = bdRegion
|
||||
// row['M'] = bdName
|
||||
// row['N'] = bdAddress
|
||||
// row['O'] = bdEstateType
|
||||
// row['P'] = bdLng
|
||||
// row['Q'] = bdLat
|
||||
}
|
||||
|
||||
},0);
|
||||
@@ -135,29 +204,32 @@ class AddThirdPriceDistrict {
|
||||
|
||||
|
||||
def exportDatas(){
|
||||
def city = "无锡"
|
||||
def datas = findMatchErrorDatas(city);
|
||||
println "共【"+datas.size()+"】条"
|
||||
datas = filterName(datas);
|
||||
println "过滤后共【"+datas.size()+"】条"
|
||||
def resMap = groupByName(datas);
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
|
||||
resMap.each {k,v->
|
||||
def ret = []
|
||||
def arrs = k.split("##")
|
||||
ret << arrs[0]
|
||||
ret << arrs[1]
|
||||
if(arrs.size()==3 && arrs[2]){
|
||||
ret << arrs[2]
|
||||
}else{
|
||||
ret << ""
|
||||
//跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞",
|
||||
def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
for(def city : citys){
|
||||
def datas = findMatchErrorDatas(city);
|
||||
println "共【"+datas.size()+"】条"
|
||||
datas = filterName(datas);
|
||||
println "过滤后共【"+datas.size()+"】条"
|
||||
def resMap = groupByName(datas);
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
|
||||
resMap.each {k,v->
|
||||
def ret = []
|
||||
def arrs = k.split("##")
|
||||
ret << arrs[0]
|
||||
ret << arrs[1]
|
||||
if(arrs.size()==3 && arrs[2]){
|
||||
ret << arrs[2]
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << resAddrMap.get(k)
|
||||
ret << v.join(",")
|
||||
rets << ret
|
||||
}
|
||||
ret << resAddrMap.get(k)
|
||||
ret << v.join(",")
|
||||
rets << ret
|
||||
ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后")
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后")
|
||||
}
|
||||
|
||||
def filterName(datas){
|
||||
@@ -465,7 +537,7 @@ class AddThirdPriceDistrict {
|
||||
def findMatchErrorDatas(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date",202005)
|
||||
q.put("s_date",new BasicDBObject('$gte', 202101))
|
||||
// q.put("name","博济聚创科技园")
|
||||
// q.put("source","贝壳找房")
|
||||
def dbcursor = matchErrorLogColl.find(q)
|
||||
@@ -499,8 +571,8 @@ class AddThirdPriceDistrict {
|
||||
return thirdPartyColl.findOne(q)
|
||||
}
|
||||
|
||||
def addInfo(){
|
||||
|
||||
def addInfo(filePath){
|
||||
province = "江西"
|
||||
// def city = "苏州"
|
||||
// def datas = findMatchErrorDatas(city);
|
||||
// println "共【"+datas.size()+"】条"
|
||||
@@ -509,8 +581,8 @@ class AddThirdPriceDistrict {
|
||||
// def resMap = groupByName(datas);
|
||||
|
||||
def index = 0;
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx", { row ->
|
||||
def flag = row['R']
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def flag = row['T']
|
||||
def dgDatas = row['G']
|
||||
// if(!flag){
|
||||
index++
|
||||
@@ -544,9 +616,9 @@ class AddThirdPriceDistrict {
|
||||
// def estateType = filterEstateTypeMap.get(row['A']+"##"+row['B']+"##"+row['C'])
|
||||
// row['W'] = estateType
|
||||
|
||||
if(!row['X']){
|
||||
def name = nameProcess(row['B'], row['C'])
|
||||
row['X'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C'])
|
||||
if(!row['Z']){
|
||||
//def name = nameProcess(row['B'], row['C'])
|
||||
row['Z'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C'])
|
||||
}
|
||||
|
||||
//
|
||||
@@ -953,11 +1025,11 @@ class AddThirdPriceDistrict {
|
||||
|
||||
def queryPoiMap(def city, def region, def name){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and adname = '" + region + "'")
|
||||
sqlBuffer.append(" and name = '" + name + "'")
|
||||
sqlBuffer.append(" and source = '高德'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
sqlBuffer.append(" and source = '高德' order by score() desc limit 1")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def map = [:]
|
||||
while (rs.next()) {
|
||||
@@ -966,8 +1038,11 @@ class AddThirdPriceDistrict {
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.type = rs.getString("type")
|
||||
map.lng = rs.getString("location.lon")
|
||||
map.lat = rs.getString("location.lat")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
@@ -982,6 +1057,37 @@ class AddThirdPriceDistrict {
|
||||
return null
|
||||
}
|
||||
|
||||
def matchDistDatas(def path){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(path,{ List row ->
|
||||
index ++
|
||||
if(!row['I'] && !row['J']){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['C'], row['D']);
|
||||
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
|
||||
//row['J'] = rMap.city
|
||||
row['U'] = rMap.region
|
||||
row['V'] = rMap.name
|
||||
row['W'] = rMap.address
|
||||
|
||||
if(rMap.lat){
|
||||
// def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
|
||||
// def maplng = baidu.x
|
||||
// def maplat = baidu.y
|
||||
// row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(row['C'], row['V'])
|
||||
if(score<0.3){
|
||||
score = CosineSimilarAlgorithm.getSimilarity(row['D'], row['V'])
|
||||
}
|
||||
row['X'] = score
|
||||
}
|
||||
}
|
||||
println index +" "+row
|
||||
}
|
||||
},0)
|
||||
println "5.小区匹配完成"
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+20
-15
@@ -7,7 +7,7 @@ import java.sql.ResultSet
|
||||
class EsSearchUtils {
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -64,24 +64,29 @@ class EsSearchUtils {
|
||||
sqlBuffer.append(" and region = '"+region+"'")
|
||||
sqlBuffer.append(" and district_name = '"+name+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" limit 100")
|
||||
sqlBuffer.append(" limit 200")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def rets = []
|
||||
while (rs.next()){
|
||||
def height = rs.getInt("height")
|
||||
def direction = rs.getString("direction")
|
||||
def fitment = rs.getString("fitment")
|
||||
def room = rs.getInt("room")
|
||||
def hall = rs.getInt("hall")
|
||||
def toilet = rs.getInt("toilet")
|
||||
def map = [:]
|
||||
map.height = height
|
||||
map.direction = direction
|
||||
map.fitment = fitment
|
||||
map.room = room
|
||||
map.hall = hall
|
||||
map.toilet = toilet
|
||||
rets << map
|
||||
if(room){
|
||||
def height = rs.getInt("height")
|
||||
def direction = rs.getString("direction")
|
||||
def fitment = rs.getString("fitment")
|
||||
def hall = rs.getInt("hall")
|
||||
def toilet = rs.getInt("toilet")
|
||||
def area = rs.getInt("area")
|
||||
def map = [:]
|
||||
map.height = height
|
||||
map.direction = direction
|
||||
map.fitment = fitment
|
||||
map.room = room
|
||||
map.hall = hall
|
||||
map.toilet = toilet
|
||||
map.area = area
|
||||
rets << map
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
@@ -259,6 +259,31 @@ class FiledStrand {
|
||||
return updateFile;
|
||||
}
|
||||
|
||||
static def processArea(all_area){
|
||||
if(all_area){
|
||||
def v = getValue(all_area)
|
||||
if(v>=1000 && v<=10000000){
|
||||
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processCarSite(car_site_count){
|
||||
def v = getValue(car_site_count.toString());
|
||||
if(v){
|
||||
if(v>=1 && v<=100000){
|
||||
try {
|
||||
return v.toString().toInteger();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def addBuildAreaToData(def dist, def updateFile){
|
||||
def build_area = null;
|
||||
if(dist.build_area){
|
||||
@@ -298,6 +323,26 @@ class FiledStrand {
|
||||
}
|
||||
return updateFile;
|
||||
}
|
||||
|
||||
public static def processCubagePercent(cubage_percent, estate_type2){
|
||||
if(cubage_percent){
|
||||
def v = getValue(cubage_percent)
|
||||
if(v){
|
||||
if(estate_type2 == "别墅"){
|
||||
if(v>=0.3 && v<=1){
|
||||
return v.toString()
|
||||
}
|
||||
}else{
|
||||
if(v>1 && v<=10){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def addVirescencePercentToData(dist, updateFile){
|
||||
def virescence_percent = null
|
||||
if(dist.virescence_percent){
|
||||
@@ -312,6 +357,16 @@ class FiledStrand {
|
||||
return updateFile;
|
||||
}
|
||||
|
||||
static def processVirescencePercent(virescence_percent){
|
||||
if(virescence_percent){
|
||||
def v = getValue(virescence_percent)
|
||||
if(v && v>15 && v<=70){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def addBuildTypeToData(dist, updateFile){
|
||||
if(dist.estateType2 && dist.buildType){
|
||||
def house = ["超高层","小高层","多层","高层","花园洋房","老洋房","四合院"]
|
||||
@@ -394,6 +449,7 @@ class FiledStrand {
|
||||
}
|
||||
return estateCharege;
|
||||
}
|
||||
|
||||
static def processHouseholdCount(def household_count, def estateType2){
|
||||
if(household_count){
|
||||
household_count = household_count.toInteger()
|
||||
|
||||
+32
-18
@@ -57,8 +57,12 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
// def prefix = "village_"
|
||||
// def categorys = ["190108","190109"]//村庄
|
||||
|
||||
def prefix = "company_"
|
||||
def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
|
||||
|
||||
def prefix = "food_"
|
||||
def categorys = ["050000"]
|
||||
|
||||
// def prefix = "company_"
|
||||
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
@@ -116,20 +120,25 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys = ["南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def allcitys = ["上海"]//CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
def stop = false
|
||||
while(stop == false){
|
||||
def cityLations = findAllLation(city);
|
||||
if(!cityLations){
|
||||
stop = true;
|
||||
}else{
|
||||
println "开始处理["+city +"]:"+cityLations.size()
|
||||
gaoDeCitySearch(cityLations);
|
||||
for(def city : allcitys){
|
||||
|
||||
if(!citys.contains(city)){
|
||||
def stop = false
|
||||
while(stop == false){
|
||||
def cityLations = findAllLation(city);
|
||||
if(!cityLations){
|
||||
stop = true;
|
||||
}else{
|
||||
println "开始处理["+city +"]:"+cityLations.size()
|
||||
gaoDeCitySearch(cityLations);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,7 +150,11 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
// q.put("status", "失败")
|
||||
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def results = []
|
||||
for(def d : dbcursor){
|
||||
results << d
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -182,14 +195,15 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
}
|
||||
|
||||
def searching(i, location){
|
||||
// def bound_gd = location.bound_gd
|
||||
// def bound_gd = location.bound_gd
|
||||
// def lats = [bound_gd[1], bound_gd[3]]
|
||||
// def lngs = [bound_gd[0], bound_gd[2]]
|
||||
// lats.sort()
|
||||
// lngs.sort()
|
||||
// println index
|
||||
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
||||
search(i, location)
|
||||
println i+" "+ location
|
||||
// 120.019556,30.313701
|
||||
//if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){
|
||||
search(i, location)
|
||||
// }
|
||||
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
|
||||
@@ -29,6 +29,8 @@ class OpenApiUtils {
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.id = dto.id
|
||||
rMap.lng = dto.lng
|
||||
rMap.lat = dto.lat
|
||||
if(dto.address && dto.address!="null"){
|
||||
rMap.address = dto.address
|
||||
}
|
||||
|
||||
@@ -45,18 +45,20 @@ class OpenUserBatch {
|
||||
}
|
||||
|
||||
def addShenZhenFjtUser(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\农行\\肇庆农行支行房价通账号.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
|
||||
def i = 0;
|
||||
for(def data : datas){
|
||||
i++;
|
||||
def id = 252580 + i;
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)";
|
||||
def id = 252608 + i;
|
||||
BigDecimal bd = new BigDecimal(data[3].toString());
|
||||
String mobile = bd.toPlainString();
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
|
||||
onLineSqlInstance.execute(sql);
|
||||
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王辰岑')";
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
|
||||
onLineSqlInstance.execute(sql2)
|
||||
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'肇庆','肇庆:')"
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
|
||||
onLineSqlInstance.execute(sql3)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
@@ -28,8 +29,9 @@ class RegionPointCrawl {
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
@@ -55,7 +57,13 @@ class RegionPointCrawl {
|
||||
}
|
||||
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
|
||||
if(ret.size()>0){
|
||||
saveData(cityMap.name, m.name, m.ad_code , ret);
|
||||
def standardCity = RegionData.formatCity(cityMap.name)
|
||||
if(standardCity){
|
||||
saveData(standardCity, m.name, m.ad_code , ret);
|
||||
}else{
|
||||
println cityMap.name +" 没有标准化"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -72,7 +80,7 @@ class RegionPointCrawl {
|
||||
for(def cityMap : cityList){
|
||||
if(cityMap.level == "city"){
|
||||
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
|
||||
saveGaoDeRegionCode(province, cityMap.districts);
|
||||
saveGaoDeRegionCode(province, cityMap.name, cityMap.districts);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,10 +89,10 @@ class RegionPointCrawl {
|
||||
}
|
||||
}
|
||||
|
||||
def saveGaoDeRegionCode(province, districts){
|
||||
def saveGaoDeRegionCode(province, cityName,districts){
|
||||
if(districts){
|
||||
for(def dist : districts){
|
||||
if(dist.level=="district"){
|
||||
if(dist.level=="district" || cityName=="东莞市"){
|
||||
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||
}
|
||||
}
|
||||
@@ -221,4 +229,3 @@ class RegionPointCrawl {
|
||||
return KeyList.get(count);
|
||||
}
|
||||
}
|
||||
H
|
||||
+36
-1
@@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
class WuXiAddressTest {
|
||||
static main(args) {
|
||||
def task = new WuXiAddressTest()
|
||||
task.run()
|
||||
//task.run()
|
||||
task.addAddress()
|
||||
}
|
||||
|
||||
def addAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d->
|
||||
def prov = d['C']
|
||||
def city = d['D']
|
||||
def region = d['E']
|
||||
def address = d['B']
|
||||
def road = d['J']
|
||||
def street = d['R']
|
||||
|
||||
def newAddress = prov+"省"+city +"市"
|
||||
if(region == "惠山"){
|
||||
newAddress = newAddress + region +"区"
|
||||
}else{
|
||||
newAddress = newAddress + region +"市"
|
||||
}
|
||||
|
||||
d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "")
|
||||
def dist = d['U']
|
||||
|
||||
def s1 = null
|
||||
if(road){
|
||||
s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","")
|
||||
newAddress = newAddress + street + dist + road + s1
|
||||
}else if(d['L']){
|
||||
s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","")
|
||||
newAddress = newAddress + street + dist + d['L'] + s1
|
||||
}
|
||||
|
||||
d['V'] = newAddress
|
||||
},0)
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -18,6 +51,8 @@ class WuXiAddressTest {
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
|
||||
if(!d['S']){
|
||||
def locationStr = getLocationInfo(d['D'], d['Q'])
|
||||
|
||||
+10
-12
@@ -6,6 +6,7 @@ import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -47,14 +48,13 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-05-01"
|
||||
static def statDate = "2021-08-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年9月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
@@ -308,10 +308,7 @@ class YouDaDataParser {
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
latch.await()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def saveEsfHouseToDB(def data){
|
||||
@@ -367,15 +364,16 @@ class YouDaDataParser {
|
||||
row.ydId = data[2].toInteger()
|
||||
|
||||
row.fjCity = city
|
||||
def map = matchDistrict(city, row.region, row.districtName)
|
||||
if(!map){
|
||||
map = matchDistrict(city, row.region, row.address)
|
||||
}
|
||||
// def map = matchDistrict(city, row.region, row.districtName)
|
||||
// if(!map){
|
||||
// map = matchDistrict(city, row.region, row.address)
|
||||
// }
|
||||
def map = SeaweedDataMatchProcess.doMatch(city, row.region, row.districtName, row.address);
|
||||
if(map){
|
||||
row.fjRegion = map.region
|
||||
row.fjName = map.name
|
||||
row.lat = map.lat
|
||||
row.lng = map.lng
|
||||
row.lat = map.lat.toDouble()
|
||||
row.lng = map.lng.toDouble()
|
||||
row.location = map.lat.toString()+","+map.lng.toString()
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(map.id.toString());
|
||||
|
||||
@@ -7,6 +7,13 @@ import java.util.regex.Pattern
|
||||
class CcbCodeUtils {
|
||||
|
||||
|
||||
//否:0 是:1
|
||||
static def parseFlag(String flag){
|
||||
if(flag && flag == "是"){
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static def parseBuildYear(String buildDate) {
|
||||
if(buildDate){
|
||||
@@ -53,6 +60,44 @@ class CcbCodeUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
// 高层塔楼 0101 增加“超高层” 0108
|
||||
// 高层板楼 0102 增加“老洋房” 0109
|
||||
// 小高层板楼 0103 增加“花园洋房” 0110
|
||||
// 多层板楼 0104 增加“四合院” 0111
|
||||
// 独栋别墅 0105 增加“老式别墅洋房” 0112
|
||||
// 联排别墅 0106
|
||||
// 叠拼别墅 0107
|
||||
// 其他建筑形式 0199
|
||||
|
||||
static def parseBuildType(def buildType){
|
||||
if(buildType){
|
||||
if(buildType.contains("老式别墅洋房")){
|
||||
return "0112"
|
||||
}else if(buildType.contains("四合院")){
|
||||
return "0111"
|
||||
}else if(buildType.contains("花园洋房")){
|
||||
return "0110"
|
||||
}else if(buildType.contains("老洋房")){
|
||||
return "0109"
|
||||
}else if(buildType.contains("叠拼")){
|
||||
return "0107"
|
||||
}else if(buildType.contains("联排")){
|
||||
return "0106"
|
||||
}else if(buildType.contains("独栋")){
|
||||
return "0105"
|
||||
}else if(buildType.contains("多层")){
|
||||
return "0104"
|
||||
}else if(buildType.contains("超高层")){
|
||||
return "0108"
|
||||
}else if(buildType.contains("小高层")){
|
||||
return "0103"
|
||||
}else if(buildType.contains("高层")){
|
||||
return "0102"
|
||||
}
|
||||
}
|
||||
return "0199"
|
||||
}
|
||||
|
||||
|
||||
//朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399
|
||||
static def getDirectionCode(def direction){
|
||||
@@ -161,4 +206,19 @@ class CcbCodeUtils {
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//房屋用途
|
||||
static def parseHouseUse(String houseUse){
|
||||
if(houseUse){
|
||||
if(houseUse.contains("住宅")){
|
||||
return 11;
|
||||
}else if(houseUse.contains("商住")){
|
||||
return 13;
|
||||
}else{
|
||||
return 99
|
||||
}
|
||||
}
|
||||
return 99
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
|
||||
class CcbXykData {
|
||||
static main(args) {
|
||||
def task = new CcbXykData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def districts = CommonDatautils.findAllDistrictByCity("ÉϺ£","district",["city","region","name","estate_charge"])
|
||||
for(def d : districts){
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+26
-16
@@ -3,35 +3,45 @@ package com.fangjia.script.gj.ccb
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FileUtils {
|
||||
static def batchCreateFile(ret, fileName, headers,charset){
|
||||
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
sb.append(headers.join("|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
if(headers){
|
||||
sb.append(headers.join("|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
}
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
println index
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
|
||||
+465
@@ -0,0 +1,465 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.AMQP
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AddDistrictFieldValue {
|
||||
|
||||
def detailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDistrictFieldValue();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
|
||||
//第三方均价数据匹配
|
||||
// for(def city : citys){
|
||||
// def rets = findAllDataByCity(city)
|
||||
// match(rets)
|
||||
// }
|
||||
|
||||
//第三方均价数据填补
|
||||
// for(def city : citys){
|
||||
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
// def rMap = getMatchDistrict(city)
|
||||
// createResultValue(dists, rMap)
|
||||
// }
|
||||
|
||||
//
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
|
||||
// addFinishDate(dists)
|
||||
addFieldValueByHouse(dists)
|
||||
}
|
||||
}
|
||||
|
||||
def match(def rets){
|
||||
def totalCount = new AtomicLong()
|
||||
def total = rets.size()
|
||||
final CountDownLatch latch = new CountDownLatch(total)
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def d = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def region = RegionData.formatRegion(d.city, d.region)
|
||||
if(region) {
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(d.city, region, d.name, d.address);
|
||||
if(rMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_city", rMap.city)
|
||||
udata.put("match_region", rMap.region)
|
||||
udata.put("match_name", rMap.name)
|
||||
detailColl.update(q, new BasicDBObject('$set',udata))
|
||||
}
|
||||
println totalCount.get()+"/${total}"+" "+d.city +" "+d.region+" "+d.name +" MATCH:"+rMap
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
def findAllDataByCity(def city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("address", 1)
|
||||
keys.put("match_name", 1)
|
||||
|
||||
def dbcursor = detailColl.find(new BasicDBObject("city", city), keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
if(!d.match_name){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def getMatchDistrict(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("match_name", new BasicDBObject('$exists', true))
|
||||
def dbcursor = detailColl.find(q)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.match_city +"#"+d.match_region+"#"+d.match_name
|
||||
def sourceMap = rMap.get(key)
|
||||
if(!sourceMap){
|
||||
sourceMap = [:]
|
||||
rMap.put(key, sourceMap)
|
||||
}
|
||||
def value = sourceMap.get(d.root)
|
||||
if(!value){
|
||||
sourceMap.put(d.root, d)
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
def addFinishDate(dists){
|
||||
def updateIndex = 0
|
||||
def total = dists.size()
|
||||
for(def d : dists){
|
||||
if(!d.finish_building_date){
|
||||
def udata = new BasicDBObject()
|
||||
def newValue = FiledStrand.processFinishBuildingDate(d.live_date)
|
||||
if(!newValue && d.open_date){
|
||||
def date = d.open_date
|
||||
newValue = date.getYear()+1900
|
||||
}
|
||||
if(!newValue && d.build_date){
|
||||
newValue = FiledStrand.processFinishBuildingDate(d.build_date)
|
||||
}
|
||||
if(newValue){
|
||||
udata.put("finish_building_date", newValue)
|
||||
}
|
||||
if(udata.size()>0){
|
||||
updateIndex++
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def createResultValue(dists, rMap){
|
||||
def updateIndex = 0
|
||||
def total = dists.size()
|
||||
for(def d : dists){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
if(sourceMap){
|
||||
def udata = new BasicDBObject()
|
||||
if(!d.address){
|
||||
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("address", lastValue)
|
||||
}
|
||||
}
|
||||
if(!d.all_area){
|
||||
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("all_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.build_area){
|
||||
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("build_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.virescence_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("virescence_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.cubage_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("cubage_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.finish_building_date){
|
||||
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("finish_building_date", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.household_count){
|
||||
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("household_count", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.car_site_count){
|
||||
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("car_site_count", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(udata.size()>0){
|
||||
updateIndex++
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getValuesByField(sourceMap, field, estateType){
|
||||
def map = [:]
|
||||
sourceMap.each{k,v->
|
||||
def oldValue = v.get(field);
|
||||
if(oldValue){
|
||||
def newValue = null;
|
||||
if(["address"].contains(field)){
|
||||
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
|
||||
}
|
||||
if(["all_area","build_area"].contains(field)){
|
||||
newValue = FiledStrand.processArea(oldValue)
|
||||
}
|
||||
if(["build_date"].contains(field)){
|
||||
newValue = FiledStrand.processFinishBuildingDate(oldValue)
|
||||
}
|
||||
if(["house_count"].contains(field)){
|
||||
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
|
||||
}
|
||||
if(["greening_rate"].contains(field)){
|
||||
newValue = FiledStrand.processVirescencePercent(oldValue)
|
||||
}
|
||||
if(["volumetric_rate"].contains(field)){
|
||||
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
|
||||
}
|
||||
if(["car_site_desc","car_site"].contains(field)){
|
||||
newValue = FiledStrand.processCarSite(oldValue)
|
||||
}
|
||||
if(newValue){
|
||||
def count = map.get(newValue)
|
||||
if(!count){
|
||||
map.put(newValue, 1)
|
||||
}else{
|
||||
map.put(newValue, count+1)
|
||||
}
|
||||
|
||||
v.put(field, newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def lastValue = null
|
||||
if(map && map.size()>0){
|
||||
map = map.sort {a, b -> b.value <=> a.value}
|
||||
def values = map.values()
|
||||
if(values.size()==1){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def count1 = map.values()[0]
|
||||
def count2 = map.values()[1]
|
||||
if(count1 > count2){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def rets = [map.keySet()[0],map.keySet()[1]]
|
||||
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
|
||||
for(def root : roots){
|
||||
def data = sourceMap.get(root)
|
||||
if(data){
|
||||
def value = data.get(field)
|
||||
if(rets.contains(value)){
|
||||
lastValue = value
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValue;
|
||||
}
|
||||
|
||||
|
||||
def addFieldValueByHouse(def dists){
|
||||
def updateIndex = new AtomicLong()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def total = dists.size()
|
||||
final CountDownLatch latch = new CountDownLatch(total)
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(d.estate_type2!="别墅" && (!d.finish_building_date || !d.build_type)){
|
||||
def udata = new BasicDBObject()
|
||||
def retMap = findHouses(d.city, d.region, "sell", d.name, d.alias)
|
||||
if(retMap){
|
||||
println retMap
|
||||
if(!d.finish_building_date && retMap.containsKey("build_date")){
|
||||
def dateMap = retMap.get("build_date")
|
||||
if(dateMap){
|
||||
def value = dateMap.keySet()[0]
|
||||
if(value){
|
||||
udata.put("finish_building_date", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!d.build_type && retMap.containsKey("height")){
|
||||
def heightMap = retMap.get("height")
|
||||
if(heightMap){
|
||||
def heights = heightMap.keySet()
|
||||
if(heights){
|
||||
def max = heights.max()
|
||||
if(max<=6){
|
||||
udata.put("build_type", "多层")
|
||||
}else if(max>6 && max<=11){
|
||||
udata.put("build_type", "小高层")
|
||||
}else if(max>11 && max<30){
|
||||
udata.put("build_type", "高层")
|
||||
}else if(max>30){
|
||||
udata.put("build_type", "超高层")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(udata.size()>0){
|
||||
updateIndex.addAndGet(1)
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
println updateIndex.get() +"/${total} "+d.city +" "+d.name+" "+udata
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{
|
||||
println totalCount.get() +" "+d.city +" "+d.name
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def name, def alias){
|
||||
|
||||
// QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
|
||||
// must(QueryBuilders.termQuery("cat", cat)).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def restMap = [:]
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-*", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
if(seawater.build_date){
|
||||
def newValue = FiledStrand.processFinishBuildingDate(seawater.build_date)
|
||||
if(newValue){
|
||||
def valueMap = restMap.get("build_date")
|
||||
if(!valueMap){
|
||||
valueMap = [:]
|
||||
restMap.put("build_date", valueMap)
|
||||
}
|
||||
def count = valueMap.get(newValue)
|
||||
if(!count){
|
||||
valueMap.put(newValue, 1)
|
||||
}else{
|
||||
valueMap.put(newValue, count + 1)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(seawater.height){
|
||||
def hightMap = restMap.get("height")
|
||||
if(!hightMap){
|
||||
hightMap = [:]
|
||||
restMap.put("height", hightMap)
|
||||
}
|
||||
def count = hightMap.get(seawater.height)
|
||||
if(!count){
|
||||
count = 1
|
||||
hightMap.put(seawater.height, count)
|
||||
}else{
|
||||
hightMap.put(seawater.height, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(restMap){
|
||||
def dateMap = restMap.get("build_date")
|
||||
if(dateMap){
|
||||
dateMap = dateMap.sort {a, b -> b.value <=> a.value}
|
||||
restMap.put("build_date", dateMap)
|
||||
}
|
||||
}
|
||||
return restMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
class BestDistrictPriceExport {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new BestDistrictPriceExport();
|
||||
//task.run()
|
||||
task.run2()
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def filePath = "d:\\数据\\广发\\楼盘.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def stat = findDistrictPrice(row['A'], row['B'], row['D'], "20210801")
|
||||
if(stat){
|
||||
row['S'] = stat.avg_price
|
||||
}
|
||||
println row['A']+" "+row['B']+" "+ row['C']+" "+row['S']
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "佛山"
|
||||
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
def rets = []
|
||||
rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"]
|
||||
def distMap = findCreditHightDistrict2(city, "20210711");
|
||||
distMap.each{region, dists->
|
||||
|
||||
if(region=="三水"){
|
||||
def addIndex = 0
|
||||
for(def dist : dists){
|
||||
if(addIndex <=30){
|
||||
def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name)
|
||||
def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711")
|
||||
if(otherPrice){
|
||||
def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price
|
||||
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
|
||||
def ret = []
|
||||
ret << "佛山"
|
||||
ret << dist.region
|
||||
ret << dist.district_name
|
||||
ret << dist.estate_type
|
||||
for(def weekend : months){
|
||||
def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend)
|
||||
if(obj){
|
||||
ret << obj.avg_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
ret << otherPrice.lianjia_esf_price
|
||||
ret << stat.credit
|
||||
rets << ret
|
||||
addIndex++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
|
||||
def findDistrictPrice(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def obj = DIST_SELL_STAT.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def findCreditHightDistrict(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def typeMap = retMaps.get(d.region)
|
||||
if(!typeMap){
|
||||
typeMap = [:]
|
||||
retMaps.put(d.region, typeMap)
|
||||
}
|
||||
|
||||
def rets = typeMap.get(d.estate_type)
|
||||
if(!rets){
|
||||
rets = []
|
||||
typeMap.put(d.estate_type, rets)
|
||||
}
|
||||
|
||||
def count = 0
|
||||
typeMap.each {k,v->
|
||||
if(v){
|
||||
count = count + v.size()
|
||||
}
|
||||
}
|
||||
|
||||
if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}else if(rets.size() <= 20){
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}
|
||||
|
||||
def total = 0
|
||||
retMaps.each {k,m->
|
||||
m.each {t,v->
|
||||
total = total + v.size()
|
||||
}
|
||||
}
|
||||
if(total>100){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
def rows = []
|
||||
retMaps.each {k,m->
|
||||
m.each {t,v->
|
||||
rows.addAll(v)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
def findCreditHightDistrict2(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
def m = [:]
|
||||
m.city = d.city
|
||||
m.region = d.region
|
||||
m.estate_type = d.estate_type
|
||||
m.district_name = d.district_name
|
||||
rets << m
|
||||
}
|
||||
//
|
||||
// def rows = []
|
||||
// retMaps.each {k,v->
|
||||
// rows.addAll(v)
|
||||
// }
|
||||
return retMaps;
|
||||
}
|
||||
|
||||
def findNearThirdPartyPrice(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", 202106)
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
return AVERAGE_STATISTICS.findOne(q)
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
|
||||
def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"]
|
||||
|
||||
def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
|
||||
for(def s : districts){
|
||||
def arrs = s.split("_")
|
||||
def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2])
|
||||
def ret = []
|
||||
ret << s
|
||||
if(m){
|
||||
ret << m.city
|
||||
ret << m.district
|
||||
ret << m.name
|
||||
if(m.alias){
|
||||
ret << m.alias.toString().replace("[","").replace("]","")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << m.address
|
||||
if(m.addressAlias){
|
||||
ret << m.addressAlias.toString().replace("[","").replace("]","")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << m.avgPrice
|
||||
ret << m.lng
|
||||
ret << m.lat
|
||||
ret << m.block
|
||||
ret << m.loopLine
|
||||
ret << m.propertyType
|
||||
ret << m.buildingType
|
||||
ret << m.commercialArea
|
||||
ret << m.totalUnitCount
|
||||
ret << m.managementFee
|
||||
ret << m.parking
|
||||
ret << m.parkingDescription
|
||||
ret << m.ownershipPeriod
|
||||
ret << m.ownershipType
|
||||
ret << m.startBuildTime
|
||||
ret << m.startSellingTime
|
||||
ret << m.finishBuildTime
|
||||
ret << m.moveinTime
|
||||
ret << m.floorSpace
|
||||
ret << m.constructionSpace
|
||||
ret << m.plotRatio
|
||||
ret << m.greenPercentage
|
||||
ret << m.preSalePermit
|
||||
ret << m.developer
|
||||
ret << m.propertyManagement
|
||||
ret << m.decorationStandard
|
||||
ret << m.buildingStructure
|
||||
ret << m.supportingFacilities
|
||||
ret << m.transportation
|
||||
ret << m.relatedInfo
|
||||
ret << m.description
|
||||
ret << m.salesOfficeAdd
|
||||
ret << m.salesPhoneNumber
|
||||
ret << m.discount
|
||||
ret << m.buildingCount
|
||||
ret << m.finishYear
|
||||
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据")
|
||||
|
||||
|
||||
|
||||
|
||||
+35
-2
@@ -14,12 +14,45 @@ class FieldStat {
|
||||
static main(args) {
|
||||
def task = new FieldStat();
|
||||
// task.run()
|
||||
task.run1()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
}
|
||||
|
||||
def run3(){
|
||||
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def dist_total = 0
|
||||
|
||||
def addr_alias_total = 0
|
||||
def rets = []
|
||||
rets << ["城市","有地址别名总量","小区总量","有地址别名小区占比"]
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "addr_alias"]);
|
||||
def alias_index = 0
|
||||
def alias_total = 0
|
||||
|
||||
for(def d : districts){
|
||||
def alias = d.addr_alias
|
||||
if(alias){
|
||||
alias_index++
|
||||
}
|
||||
}
|
||||
def precent = 0
|
||||
if(districts.size()){
|
||||
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
|
||||
}
|
||||
rets << [city, alias_index, districts.size(), precent+"%"]
|
||||
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\全国城市地址别名统计.xlsx", rets, "统计")
|
||||
// println dist_total
|
||||
// println alias_total
|
||||
// println addr_alias_total
|
||||
}
|
||||
|
||||
|
||||
def run1(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def dist_total = 0
|
||||
|
||||
def addr_alias_total = 0
|
||||
|
||||
+18
-6
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -10,22 +11,23 @@ import com.mongodb.DBCollection
|
||||
class RegionPriceExport {
|
||||
|
||||
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
|
||||
DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new RegionPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def months = [202010,202011,202012,202101,202102,202103]
|
||||
def regions = CommonUtils.getAllRegions("福州","region")
|
||||
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
def regions = CommonUtils.getAllRegions("佛山","region")
|
||||
def rets = []
|
||||
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
|
||||
rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
regions.each{region,values->
|
||||
def data = []
|
||||
data << "福州"
|
||||
data << "佛山"
|
||||
data << region
|
||||
for(def month :months){
|
||||
def dbo = findPrice("福州", region, month);
|
||||
def dbo = findWeekendPrice("佛山", region, month);
|
||||
if(dbo){
|
||||
data << dbo.avg_price
|
||||
}else{
|
||||
@@ -34,7 +36,7 @@ class RegionPriceExport {
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
|
||||
ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
def findPrice(city, region, month){
|
||||
@@ -45,4 +47,14 @@ class RegionPriceExport {
|
||||
def dbo = REGION_SELL_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def findWeekendPrice(city, region, month){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("cat", "区域")
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
|
||||
def dbo = REGION_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
}
|
||||
|
||||
+7
-10
@@ -24,10 +24,10 @@ class DataProcess {
|
||||
|
||||
static main(args) {
|
||||
//建行信用卡城市(临汾--晋城).xlsx
|
||||
def filePath = "D:\\数据\\建行银行\\建行信用卡入库小区V1.xlsx"
|
||||
def filePath = "D:\\数据\\小区\\地图\\兰州.xlsx"
|
||||
def task = new DataProcess();
|
||||
// task.batchSaveDataToDB(filePath)
|
||||
// task.mapBatchSaveDataToDB(filePath)
|
||||
task.mapBatchSaveDataToDB(filePath)
|
||||
// task.calcutePrice(filePath);
|
||||
// task.calcutePrice2();
|
||||
// task.deleteRepDist("株洲");
|
||||
@@ -35,7 +35,7 @@ class DataProcess {
|
||||
|
||||
|
||||
// task.standartName(filePath);
|
||||
task.saveGaoDeMap(filePath);
|
||||
// task.saveGaoDeMap(filePath);
|
||||
|
||||
// task.updateDBAddress();
|
||||
// task.updateRegion()
|
||||
@@ -120,13 +120,10 @@ class DataProcess {
|
||||
def lastDatas = []
|
||||
if(datas){
|
||||
def keys = []
|
||||
for(def data : datas){
|
||||
if(data['A']=="厦门" && data['I'] && data['I'].toString().startsWith("24")){
|
||||
for(def data : datas){//20,21,25,
|
||||
if(data['I'] && [20,21,26,27].contains(data['I'].toString().toDouble().toInteger())){
|
||||
def key = data['A']+"##"+data['B']+"##"+data['H']
|
||||
if(!keys.contains(key) && data['H'].size()>2){
|
||||
// if(data['I'].toInteger()==22){//&& data['N']<3
|
||||
// continue;
|
||||
// }
|
||||
if(!keys.contains(key) && data['H'] && data['H'].size()>2){
|
||||
keys << key
|
||||
lastDatas << data
|
||||
}
|
||||
@@ -168,7 +165,7 @@ class DataProcess {
|
||||
addIndex++
|
||||
def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null)
|
||||
println "${addIndex}【新增入库】"+ values
|
||||
// FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
|
||||
}else{
|
||||
println "【已存在】 ${d['A']} ${d['B']} ${d['H']}"
|
||||
}
|
||||
|
||||
+26
-20
@@ -1,8 +1,11 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
@@ -13,7 +16,7 @@ class ExportData {
|
||||
|
||||
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -26,6 +29,7 @@ class ExportData {
|
||||
def task = new ExportData();
|
||||
task.exportThird()
|
||||
// task.exportMap();
|
||||
|
||||
}
|
||||
|
||||
def exportMap(){
|
||||
@@ -51,19 +55,18 @@ class ExportData {
|
||||
//"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特"
|
||||
//宁夏 ----- 中卫 固原 石嘴山 吴忠 银川
|
||||
//青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁
|
||||
def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
|
||||
"中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
|
||||
"防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
|
||||
//"沈阳","大连","抚顺","鞍山"
|
||||
//"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆"
|
||||
//"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛",
|
||||
//"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州",
|
||||
//"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城",
|
||||
// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
|
||||
// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
|
||||
// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
|
||||
def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"];
|
||||
|
||||
for(def city : citys){
|
||||
def path = "D:\\数据\\小区\\地图\\${city}.xlsx";
|
||||
def datas = findAllDatas(city, null)
|
||||
println "${city}共计:"+datas.size()
|
||||
ExcelUtils.write(path, datas, "数据")
|
||||
}
|
||||
|
||||
def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx";
|
||||
def datas = findAllDatas(citys, null)
|
||||
println "共计:"+datas.size()
|
||||
ExcelUtils.write(path, datas, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -71,7 +74,7 @@ class ExportData {
|
||||
|
||||
|
||||
def exportThird(){
|
||||
def citys = ["重庆"]
|
||||
def citys = ["南昌"]
|
||||
for(def city : citys){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","别名"]
|
||||
@@ -98,9 +101,9 @@ class ExportData {
|
||||
|
||||
|
||||
|
||||
def findAllDatas(def citys, def pname){
|
||||
def findAllDatas(def city, def pname){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'")
|
||||
// sqlBuffer.append(" and pname = '"+pname+"'")
|
||||
// sqlBuffer.append(" and cityname = '"+pname+"'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
@@ -113,16 +116,19 @@ class ExportData {
|
||||
if(source && source=="高德"){
|
||||
println index
|
||||
def type = rs.getObject("type")
|
||||
def city = rs.getString("cityname")
|
||||
if(citys.contains(city) && type.contains("商务住宅")){
|
||||
//def city = rs.getString("cityname")
|
||||
if(type.contains("住宅")){
|
||||
def ret = []
|
||||
ret << city
|
||||
ret << rs.getString("adname")
|
||||
ret << rs.getString("name")
|
||||
ret << rs.getString("address")
|
||||
ret << rs.getString("type")
|
||||
ret << rs.getFloat("location.lon")
|
||||
ret << rs.getFloat("location.lat")
|
||||
def loaction = rs.getObject("location")
|
||||
if(loaction){
|
||||
ret << DataHandleUtils.getDoubleRound(6, loaction.x)
|
||||
ret << DataHandleUtils.getDoubleRound(6, loaction.y)
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -68,13 +68,16 @@ class FangjiaClientUtils {
|
||||
|
||||
public static def addressStand(def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
|
||||
+180
-29
@@ -1,51 +1,65 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/***
|
||||
* 地图小区处理
|
||||
*/
|
||||
class MapDistrictProcess {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
|
||||
static main(args) {
|
||||
def task = new MapDistrictProcess();
|
||||
task.run()
|
||||
// task.updateRegion()
|
||||
}
|
||||
|
||||
/**
|
||||
* 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路
|
||||
* 20ㄩ祥肮Е郖陔崝ㄗ踱笢拸ㄘ 21ㄩ煦ヽ陔崝 22:煦杅婓0.6眕奻
|
||||
* 20:不同区域新增(库中无) 21:分期新增 23:不同区域,距离相近 25:无匹配,可新增 22:有匹配,距离1000米以下,相似度在0.6以上 24:疑似匹配有误 27:人工判定入库
|
||||
* @return
|
||||
*/
|
||||
def run(){
|
||||
//"蔬劼華芞.xlsx","涳蔬華芞.xlsx","腦膘華芞.xlsx","蔬昹華芞.xlsx","刓陲華芞.xlsx","假閣華芞.xlsx",
|
||||
//"奻漆華芞.xlsx","控儔華芞.xlsx"
|
||||
//"假閣華芞.xlsx","蔬劼華芞.xlsx","涳蔬華芞.xlsx","腦膘華芞.xlsx","蔬昹華芞.xlsx","刓陲華芞.xlsx",
|
||||
|
||||
//"笭ヰ華芞.xlsx","毞踩華芞.xlsx",
|
||||
//"奻漆華芞.xlsx","控儔華芞.xlsx",
|
||||
//"碩控華芞.xlsx","碩鰍華芞.xlsx","綬控華芞.xlsx","綬鰍華芞.xlsx","嫘昹華芞.xlsx",
|
||||
//"憚輿華芞.xlsx","賽譴華芞.xlsx""賽譴華芞.xlsx","憚輿華芞.xlsx","窪韓蔬華芞.xlsx","刓昹華芞.xlsx","漆鰍華芞.xlsx"]
|
||||
def files = ["ヤ漆華芞.xlsx","譴狦華芞.xlsx","囀蟹嘉華芞.xlsx","堁鰍華芞.xlsx","匟昹華芞.xlsx","陔蔭華芞.xlsx","侐捶華芞.xlsx"]
|
||||
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
|
||||
//"乌鲁木齐.xlsx","烟台.xlsx","保定.xlsx","贵阳.xlsx","湖州.xlsx","惠州.xlsx","吉林.xlsx","遵义.xlsx","宜昌.xlsx","芜湖.xlsx","兰州.xlsx"
|
||||
def files = ["兰州.xlsx"]
|
||||
for(def fileName : files){
|
||||
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
|
||||
typeFilter(filePath)
|
||||
nameStandard(filePath);
|
||||
dataFilter(filePath)
|
||||
matchDistDatas(filePath)
|
||||
houseCalculate(filePath);
|
||||
////// //苀數岆瘁湔婓嗣跺Е郖
|
||||
//统计是否存在多个区域
|
||||
matchDistDatas(filePath)
|
||||
moreRegionCalculate(filePath)
|
||||
//
|
||||
addFlag(filePath)
|
||||
mapMatch(filePath);
|
||||
baiduTypeFilter(filePath);
|
||||
baiduMap(filePath);
|
||||
// baiduMap(filePath);
|
||||
distanceFilter(filePath)
|
||||
predictType(filePath)
|
||||
noMatchFlag(filePath)
|
||||
matchFlag(filePath)
|
||||
addFeature(filePath)
|
||||
houseMatch(filePath);
|
||||
predictType(filePath)
|
||||
// nameScore(filePath);
|
||||
|
||||
nameScore(filePath);
|
||||
thirdPartyData(filePath)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -59,7 +73,12 @@ class MapDistrictProcess {
|
||||
row['I'] = 1
|
||||
}else{
|
||||
if(name.toString().length()<3){
|
||||
row['I'] = 18
|
||||
if(row['N'] && row['N']>=10 && !row['J']){
|
||||
row['I'] = 25
|
||||
}else{
|
||||
row['I'] = 18
|
||||
}
|
||||
|
||||
}else{
|
||||
def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
|
||||
if(s1){
|
||||
@@ -74,7 +93,7 @@ class MapDistrictProcess {
|
||||
}
|
||||
|
||||
});
|
||||
println "2.靡備徹薦俇傖"
|
||||
println "3.名称过滤完成"
|
||||
}
|
||||
|
||||
def typeFilter(filePath){
|
||||
@@ -88,15 +107,30 @@ class MapDistrictProcess {
|
||||
println "1.类型过滤完成"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def baiduTypeFilter(filePath){
|
||||
def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"]
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def type = row['S']
|
||||
if(type && !types.contains(type)){
|
||||
if(type && row['Q'] && row['Q']==row['H'] && !types.contains(type)){
|
||||
row['I'] = 1
|
||||
}
|
||||
});
|
||||
println "1.濬倰徹薦俇傖"
|
||||
println "9.百度类型过滤完成"
|
||||
}
|
||||
|
||||
def distanceFilter(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(!row['I']){
|
||||
if(row['X'] && row['Y']){
|
||||
if(row['X']<=500 && row['Y']>=0.5){
|
||||
row['I'] = 12
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
println "10.距离过滤完成"
|
||||
}
|
||||
|
||||
def nameStandard(filePath){
|
||||
@@ -109,7 +143,7 @@ class MapDistrictProcess {
|
||||
}
|
||||
|
||||
}, 0);
|
||||
println "1.靡備梓袧趙俇傖"
|
||||
println "2.名称标准化完成"
|
||||
}
|
||||
|
||||
def matchDistDatas(def path){
|
||||
@@ -117,17 +151,29 @@ class MapDistrictProcess {
|
||||
ExcelUtils.multiThreadingModify(path,{ List row ->
|
||||
index ++
|
||||
if(!row['I'] && !row['J']){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],);
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
|
||||
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
|
||||
row['J'] = rMap.city
|
||||
row['K'] = rMap.region
|
||||
row['L'] = rMap.name
|
||||
row['M'] = rMap.address
|
||||
|
||||
if(rMap.lat){
|
||||
def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
|
||||
def maplng = baidu.x
|
||||
def maplat = baidu.y
|
||||
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
|
||||
if(score<0.3){
|
||||
score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['M'])
|
||||
}
|
||||
row['Y'] = score
|
||||
}
|
||||
}
|
||||
println index +" "+row
|
||||
}
|
||||
},0)
|
||||
println "3.苤Е⑤饜俇傖"
|
||||
println "5.小区匹配完成"
|
||||
}
|
||||
|
||||
def houseCalculate(filePath){
|
||||
@@ -136,6 +182,7 @@ class MapDistrictProcess {
|
||||
row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H'])
|
||||
}
|
||||
});
|
||||
println "4.房源统计完成"
|
||||
}
|
||||
|
||||
def moreRegionCalculate(filePath){
|
||||
@@ -156,8 +203,12 @@ class MapDistrictProcess {
|
||||
}
|
||||
}
|
||||
if(!regions.contains(row['B'])){
|
||||
row['O'] = regions.toString()
|
||||
row['I'] = 20
|
||||
if(row['X'] && row['X']<=500 && row['Y'] && row['Y']>=0.5){
|
||||
row['I'] = 23
|
||||
}else{
|
||||
row['O'] = regions.toString()
|
||||
row['I'] = 20
|
||||
}
|
||||
}else if(regions.contains(row['B'])){
|
||||
row['O'] = "已存在"
|
||||
row['I'] = 10
|
||||
@@ -165,11 +216,12 @@ class MapDistrictProcess {
|
||||
}
|
||||
}
|
||||
});
|
||||
println "6.区域过滤完成"
|
||||
}
|
||||
|
||||
def addFlag(def filePath){
|
||||
def index = 0
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
index++
|
||||
if(!row['I']) {
|
||||
//根据别名查找
|
||||
@@ -201,6 +253,8 @@ class MapDistrictProcess {
|
||||
println index + " " + row
|
||||
}
|
||||
});
|
||||
|
||||
println "7.打标记完成"
|
||||
}
|
||||
|
||||
def baiduMap(def filePath){
|
||||
@@ -228,6 +282,7 @@ class MapDistrictProcess {
|
||||
}
|
||||
}
|
||||
});
|
||||
println "11.特征过滤完成"
|
||||
}
|
||||
|
||||
def mapMatch(def filePath){
|
||||
@@ -255,16 +310,22 @@ class MapDistrictProcess {
|
||||
println index +" "+row
|
||||
},0);
|
||||
|
||||
println "4.華芞⑤饜俇傖"
|
||||
println "8.地图匹配完成"
|
||||
}
|
||||
|
||||
def houseMatch(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(!row['I']) {
|
||||
def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
|
||||
if(rMap){
|
||||
row['I'] = rMap.flag
|
||||
}
|
||||
// def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
|
||||
// if(rMap){
|
||||
// row['I'] = rMap.flag
|
||||
// }
|
||||
|
||||
if(row['J']){
|
||||
if(row['X'] && row['Y'] ){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
@@ -309,6 +370,41 @@ class MapDistrictProcess {
|
||||
},0);
|
||||
}
|
||||
|
||||
def noMatchFlag(filePath) {
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if (!row['I'] && !row['J']) {
|
||||
if(row['N']>0){
|
||||
row['I'] = 25
|
||||
}else{
|
||||
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
|
||||
row['I'] = 25
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def matchFlag(filePath) {
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if (!row['I'] && row['J'] && row['X'] && row['Y']) {
|
||||
if(row['X']<=1000 && row['Y']>=0.6){
|
||||
row['I'] = 22
|
||||
}else if(row['X']<=600 && row['Y']>=0.5){
|
||||
row['I'] = 22
|
||||
}else if(row['X']<=1300 && row['Y']>=0.8){
|
||||
row['I'] = 22
|
||||
}else{
|
||||
row['I'] = 24
|
||||
}
|
||||
}else if(!row['I'] && row['J'] && row['Y']==0){
|
||||
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
|
||||
row['I'] = 26
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def nameScore(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(!row['I']) {
|
||||
@@ -338,4 +434,59 @@ class MapDistrictProcess {
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def updateRegion(){
|
||||
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx";
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def region = getRegionByLocation(row['F'], row['G']);
|
||||
if(region){
|
||||
row['B'] = RegionData.formatRegion(row['A'], region)
|
||||
}
|
||||
|
||||
},0)
|
||||
|
||||
}
|
||||
def thirdPartyData(filePath){
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(row['I'] && [20,21,25,26,27].contains(row['I'].toString().toDouble().toInteger())){
|
||||
def rets = findIsExistsOtherWebsite(row['A'], row['B'], row['H'])
|
||||
if(rets){
|
||||
row['W'] = rets.join(",")
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def findIsExistsOtherWebsite(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("region",region)
|
||||
q.put("name", name)
|
||||
def dbcursor = districtDetailColl.find(q)
|
||||
def rets = []
|
||||
for(def r : dbcursor){
|
||||
if(!rets.contains(r.root)){
|
||||
rets << r.root
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getRegionByLocation(lng, lat){
|
||||
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
|
||||
|
||||
if(jsonObject && jsonObject.info=="OK"){
|
||||
def compent = jsonObject.regeocode.addressComponent
|
||||
if(compent){
|
||||
def township = compent.township
|
||||
return township;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+14
-12
@@ -7,7 +7,9 @@ import java.util.regex.Pattern
|
||||
class NameProcess {
|
||||
|
||||
static def filterName(def name){
|
||||
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心","养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会"]
|
||||
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心",
|
||||
"养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房",
|
||||
"合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼","养生"]
|
||||
def isAdd = true;
|
||||
for(def key : keys){
|
||||
if(name.contains(key)){
|
||||
@@ -17,7 +19,7 @@ class NameProcess {
|
||||
}
|
||||
|
||||
if(isAdd){
|
||||
def endKeys = ["路","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍"]
|
||||
def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门","学校","公司"]
|
||||
for(def key : endKeys){
|
||||
if(name.toString().endsWith(key)){
|
||||
isAdd = false;
|
||||
@@ -27,7 +29,7 @@ class NameProcess {
|
||||
}
|
||||
|
||||
if(isAdd){
|
||||
def startKeys = ["FQ"]
|
||||
def startKeys = ["FQ","社区"]
|
||||
for(def key : startKeys){
|
||||
if(name.toString().endsWith(key)||name.toString().startsWith(key)){
|
||||
isAdd = false;
|
||||
@@ -36,7 +38,7 @@ class NameProcess {
|
||||
}
|
||||
}
|
||||
if(isAdd){
|
||||
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房"]
|
||||
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房", "家属宿舍楼","卫生生活区"]
|
||||
for(def str : strs){
|
||||
if(str == name){
|
||||
isAdd = false;
|
||||
@@ -56,7 +58,7 @@ class NameProcess {
|
||||
}
|
||||
}
|
||||
//.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号")
|
||||
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
|
||||
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
|
||||
name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","")
|
||||
if(name && name.contains("区")){
|
||||
def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区"))
|
||||
@@ -105,7 +107,7 @@ class NameProcess {
|
||||
}
|
||||
}
|
||||
if(name && !name.contains("路") && !name.contains("街") && !name.contains("大道") && !name.contains("巷")){
|
||||
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(号|排)"))
|
||||
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(排)"))
|
||||
if(h1){
|
||||
name = name.replaceAll(h1,"")
|
||||
}
|
||||
@@ -191,12 +193,12 @@ class NameProcess {
|
||||
if(str == name){
|
||||
return null
|
||||
}
|
||||
if(name){
|
||||
s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
|
||||
if(s1){
|
||||
return null
|
||||
}
|
||||
}
|
||||
// if(name){
|
||||
// s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
|
||||
// if(s1){
|
||||
// return null
|
||||
// }
|
||||
// }
|
||||
|
||||
return name.toUpperCase()
|
||||
}
|
||||
|
||||
+21
-13
@@ -15,36 +15,44 @@ class SeaweedDataMatchProcess {
|
||||
if(!rMap.city){
|
||||
def obj = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
if(!obj){
|
||||
obj = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
def datas = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
if(datas){
|
||||
obj = datas[0]
|
||||
}
|
||||
}
|
||||
if(obj){
|
||||
rMap.estate_type2 = obj.estate_type2
|
||||
rMap.city = obj.city
|
||||
rMap.region = obj.region
|
||||
rMap.alias = obj.alias
|
||||
rMap.name = obj.name
|
||||
rMap.lat = obj.lat/10000000000.0
|
||||
rMap.lng = obj.lng/10000000000.0
|
||||
rMap.id = obj._id.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && address){
|
||||
// if (findPattern(address)){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
// }
|
||||
}
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
||||
}
|
||||
|
||||
|
||||
|
||||
if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
||||
if (obj) {
|
||||
rMap.estate_type2 = obj.estate_type2
|
||||
if(!rMap && address){
|
||||
def flag = address.matches('^[\u4e00-\u9fa5]+?(·|´óµÀ|½Ö).+[ŪºÅ]$')
|
||||
if(flag){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
}
|
||||
}else{
|
||||
rMap = null
|
||||
}
|
||||
|
||||
// if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
||||
// if (obj) {
|
||||
// rMap.estate_type2 = obj.estate_type2
|
||||
// }
|
||||
// }else{
|
||||
// rMap = null
|
||||
// }
|
||||
if(!rMap && name && name.contains("¡¤")){
|
||||
def newName = trimChar(name).replace("¡¤","")
|
||||
rMap = matchFromDB2(city,newName)
|
||||
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class SourceDistDataMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new SourceDistDataMatch();
|
||||
// task.run()
|
||||
task.mapRun()
|
||||
// task.filterName()
|
||||
// task.filterType()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(obj){
|
||||
d['R'] = 1
|
||||
}else{
|
||||
def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709)
|
||||
if(distCount){
|
||||
d['W'] = distCount
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def mapRun(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N'] && !d['F']){
|
||||
def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']]
|
||||
def locationMap = matchMap(dist)
|
||||
if(locationMap){
|
||||
d['F'] = locationMap.source
|
||||
d['G'] = locationMap.region
|
||||
d['H'] = locationMap.mapName
|
||||
d['I'] = locationMap.address
|
||||
d['J'] = locationMap.type
|
||||
d['K'] = locationMap.lng
|
||||
d['L'] = locationMap.lat
|
||||
d['M'] = locationMap.score
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def filterName(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N']) {
|
||||
def name = d['C']
|
||||
if (name) {
|
||||
def flag = NameProcess.filterName(name)
|
||||
if (!flag) {
|
||||
d['N'] = 4
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def filterType(){
|
||||
def path = "D:\\数据\\小区\\上海第三方均价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['N']){
|
||||
def type = d['J']
|
||||
if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){
|
||||
|
||||
}else if(type && type.contains("商务住宅") && !type.contains("商住两用")){
|
||||
def count = 0
|
||||
if(d['S'] && d['S']!='null'){
|
||||
count = d['S'].toDouble().toInteger()
|
||||
}
|
||||
if(count<10){
|
||||
d['N'] = 3
|
||||
}
|
||||
}else{
|
||||
if(type){
|
||||
d['N'] = 3
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def matchMap(def dist){
|
||||
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
|
||||
if(!gaodeRest && dist.address){
|
||||
gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德")
|
||||
}
|
||||
def gdlocationMap = null
|
||||
def bdlocationMap = null
|
||||
if(gaodeRest){
|
||||
gdlocationMap = getMapLocation2(dist, gaodeRest);
|
||||
if(gdlocationMap){
|
||||
gdlocationMap.source = "高德"
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
if(baiduRest){
|
||||
bdlocationMap = getMapLocation2(dist, baiduRest);
|
||||
if(bdlocationMap){
|
||||
bdlocationMap.source = "百度"
|
||||
}
|
||||
}
|
||||
|
||||
if(gdlocationMap && bdlocationMap){
|
||||
if(gdlocationMap.score>=bdlocationMap.score){
|
||||
return gdlocationMap
|
||||
}else{
|
||||
return bdlocationMap
|
||||
}
|
||||
}else if(gdlocationMap){
|
||||
return gdlocationMap
|
||||
}else if(bdlocationMap){
|
||||
return bdlocationMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
def getMapLocation2(def dist, def rets){
|
||||
def data1 = findSimilarData(dist, rets, true)
|
||||
def data2 = findSimilarData(dist, rets, false)
|
||||
println data1
|
||||
println data2
|
||||
if(data1 && data1.score>1){
|
||||
return data1
|
||||
}else if(data2 && data2.score>1){
|
||||
return data2
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findSimilarData(def dist, def rets, def regionImpr){
|
||||
def lat = null
|
||||
def lng = null
|
||||
def region = null
|
||||
def mapName = null
|
||||
def address = null
|
||||
def score = 0
|
||||
def type = null
|
||||
def maxScore = 0
|
||||
def standardName = null
|
||||
for(def r : rets){
|
||||
if(!r.city){
|
||||
continue
|
||||
}
|
||||
if(!r.city.contains(dist.city)){
|
||||
continue
|
||||
}
|
||||
if(["公交车站"].contains(r.type)){
|
||||
continue
|
||||
}
|
||||
if(regionImpr && !r.district){
|
||||
continue
|
||||
}
|
||||
if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){
|
||||
continue
|
||||
}
|
||||
|
||||
def name = nameReplace(r.name, dist.city)
|
||||
|
||||
def distName = nameReplace(dist.name, dist.city)
|
||||
|
||||
def score3 = 0.0
|
||||
if(r.district && dist.region && r.district.contains(dist.region)){
|
||||
score3 = 0.2
|
||||
}
|
||||
|
||||
def score1 = 0.0;
|
||||
def score2 = 0.0
|
||||
if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){
|
||||
if(r.address){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2
|
||||
}
|
||||
}else{
|
||||
score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2;
|
||||
if(dist.address && r.address){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2
|
||||
}
|
||||
}
|
||||
|
||||
score3 = score1 + score2
|
||||
if(r.type && r.type.contains("住宅")){
|
||||
score3 = score3 + 0.2
|
||||
}
|
||||
if(score3 >maxScore){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
standardName = name
|
||||
address = r.address
|
||||
type = r.type
|
||||
score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10)
|
||||
maxScore = score3
|
||||
}
|
||||
println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3
|
||||
|
||||
}
|
||||
if(lng){
|
||||
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def nameReplace(def s, city){
|
||||
if(s.contains("(") && s.contains(")")){
|
||||
def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")"))
|
||||
if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){
|
||||
s = s.toString().substring(0, s.indexOf("("))
|
||||
}
|
||||
}
|
||||
def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim()
|
||||
if(name.startsWith(city)){
|
||||
def newName = name.replace(city+"市","").replace(city,"")
|
||||
if(newName && newName=="花园"){
|
||||
return name
|
||||
}else{
|
||||
name = newName
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -49,7 +49,7 @@ class ZsPersonProcess {
|
||||
// def loactionMap = task.matchMap(dist)
|
||||
// println loactionMap
|
||||
|
||||
// task.matchMap()
|
||||
task.matchMap()
|
||||
|
||||
// task.matchDb()
|
||||
// task.matchDb2()
|
||||
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
package com.fangjia.script.gj.districtwash
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
|
||||
class AddDetailDistrict {
|
||||
|
||||
def detailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDetailDistrict();
|
||||
// task.run();
|
||||
// task.delRepeatDist
|
||||
task.addEstateType()
|
||||
}
|
||||
|
||||
def addEstateType(){
|
||||
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]
|
||||
for(def city : citys){
|
||||
def keys = []
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","estate_type2"])
|
||||
for(def d : districts) {
|
||||
if(!d.estate_type2){
|
||||
def estateType2 = "普通住宅"
|
||||
if(d.name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(d.name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
DBObject update = new BasicDBObject();
|
||||
update.put('$set', new BasicDBObject("estate_type2", estateType2).append("m_date", new Date()));
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
|
||||
}
|
||||
}
|
||||
println city
|
||||
}
|
||||
}
|
||||
|
||||
def delRepeatDist(){
|
||||
def citys = CcbCommonUtils.citys
|
||||
for(def city : citys){
|
||||
def keys = []
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index++
|
||||
def key = d.city +"##"+d.region+"##"+d.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}else{
|
||||
println index +" "+d.region+" "+d.name
|
||||
DBObject update = new BasicDBObject();
|
||||
update.put('$set', new BasicDBObject("status", 1).append("m_date", new Date()));
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = ["沈阳", "哈尔滨", "吉林", "遵义", "淄博", "西宁", "青岛", "大连", "宁波"]//CcbCommonUtils.citys;
|
||||
//第三方均价数据匹配
|
||||
for (def city : citys) {
|
||||
def rets = findAllDataByCity(city)
|
||||
def rMap = getMatchDistrict(city)
|
||||
toDB(rets, rMap)
|
||||
}
|
||||
}
|
||||
|
||||
def toDB(def rets, def rMap){
|
||||
def saveDistKeys = []
|
||||
def total = rets.size()
|
||||
def index = 0
|
||||
def addIndex = 0
|
||||
for(def d : rets){
|
||||
index++
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
def name = NameProcess.nameStandard(d.city, newRegion, d.name)
|
||||
if(!name){
|
||||
continue
|
||||
}
|
||||
def flag = NameProcess.filterName(name)
|
||||
def key = null
|
||||
if(newRegion){
|
||||
key = d.city+"#"+newRegion+"#"+name
|
||||
}
|
||||
if(flag && !saveDistKeys.contains(key)){
|
||||
def gdData = FangjiaClientUtils.getMapData(d.city, newRegion, name, d.address,"", "高德")
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(gdData){
|
||||
|
||||
if(!newRegion){
|
||||
newRegion = RegionData.formatRegion(d.city, gdData.district)
|
||||
if(newRegion){
|
||||
key = d.city+"#"+newRegion+"#"+name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(key && gdData && gdData.lng){
|
||||
def matchMap = SeaweedDataMatchProcess.doMatch(d.city, newRegion, name, d.address);
|
||||
if(!matchMap){
|
||||
addIndex++
|
||||
if(gdData.lng){
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(gdData.lng.toDouble(), gdData.lat.toDouble())
|
||||
lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
|
||||
lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
|
||||
lng = (gdData.lng.toDouble()* 10000000000).longValue()
|
||||
lat = (gdData.lat.toDouble()* 10000000000).longValue()
|
||||
}
|
||||
def newAddress = DataUtil.formateAddress(d.city, d.address)
|
||||
def udata = createFieldValue(d, rMap)
|
||||
udata.put("lng", lng)
|
||||
udata.put("lat", lat)
|
||||
println index+"/${addIndex}/${total} "+d.city+" "+newRegion+" "+name+" "+d.root+" "+udata
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", name)
|
||||
udata.put("address", newAddress)
|
||||
|
||||
udata.put("lng2", lng2)
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("status", 0)
|
||||
udata.put("cat", "district")
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.save(udata)
|
||||
saveDistKeys << key
|
||||
}
|
||||
}
|
||||
// println d.city+" "+ d.region+" "+d.name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def createFieldValue(d, rMap){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
def udata = new BasicDBObject()
|
||||
if(sourceMap){
|
||||
if(!d.address){
|
||||
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("address", lastValue)
|
||||
}
|
||||
}
|
||||
if(!d.all_area){
|
||||
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("all_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.build_area){
|
||||
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("build_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.virescence_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("virescence_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.cubage_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("cubage_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.finish_building_date){
|
||||
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("finish_building_date", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.household_count){
|
||||
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("household_count", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.car_site_count){
|
||||
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("car_site_count", lastValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
return udata
|
||||
}
|
||||
|
||||
def getValuesByField(sourceMap, field, estateType){
|
||||
def map = [:]
|
||||
sourceMap.each{k,v->
|
||||
def oldValue = v.get(field);
|
||||
if(oldValue){
|
||||
def newValue = null;
|
||||
if(["address"].contains(field)){
|
||||
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
|
||||
}
|
||||
if(["all_area","build_area"].contains(field)){
|
||||
newValue = FiledStrand.processArea(oldValue)
|
||||
}
|
||||
if(["build_date"].contains(field)){
|
||||
newValue = FiledStrand.processFinishBuildingDate(oldValue)
|
||||
}
|
||||
if(["house_count"].contains(field)){
|
||||
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
|
||||
}
|
||||
if(["greening_rate"].contains(field)){
|
||||
newValue = FiledStrand.processVirescencePercent(oldValue)
|
||||
}
|
||||
if(["volumetric_rate"].contains(field)){
|
||||
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
|
||||
}
|
||||
if(["car_site_desc","car_site"].contains(field)){
|
||||
newValue = FiledStrand.processCarSite(oldValue)
|
||||
}
|
||||
if(newValue){
|
||||
def count = map.get(newValue)
|
||||
if(!count){
|
||||
map.put(newValue, 1)
|
||||
}else{
|
||||
map.put(newValue, count+1)
|
||||
}
|
||||
|
||||
v.put(field, newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def lastValue = null
|
||||
if(map && map.size()>0){
|
||||
map = map.sort {a, b -> b.value <=> a.value}
|
||||
def values = map.values()
|
||||
if(values.size()==1){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def count1 = map.values()[0]
|
||||
def count2 = map.values()[1]
|
||||
if(count1 > count2){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def rets = [map.keySet()[0],map.keySet()[1]]
|
||||
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
|
||||
for(def root : roots){
|
||||
def data = sourceMap.get(root)
|
||||
if(data){
|
||||
def value = data.get(field)
|
||||
if(rets.contains(value)){
|
||||
lastValue = value
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValue;
|
||||
}
|
||||
|
||||
|
||||
def getMatchDistrict(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("match_name", new BasicDBObject('$exists', true))
|
||||
def dbcursor = detailColl.find(q)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
if(!sourceMap){
|
||||
sourceMap = [:]
|
||||
rMap.put(key, sourceMap)
|
||||
}
|
||||
def value = sourceMap.get(d.root)
|
||||
if(!value){
|
||||
sourceMap.put(d.root, d)
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def findAllDataByCity(def city){
|
||||
def q = new BasicDBObject("city", city);
|
||||
q.put("match_name", new BasicDBObject('$exists', false))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
|
||||
def dbcursor = detailColl.find(q).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-1
@@ -1,8 +1,9 @@
|
||||
package com.fangjia.script.gj
|
||||
package com.fangjia.script.gj.districtwash
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.datastore.FieldValidatorUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.Utils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
+26
-28
@@ -19,9 +19,9 @@ class EsMonthHouseStat {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
// task.tradeRun()
|
||||
// task.houseRun()
|
||||
task.houseRun()
|
||||
|
||||
task.statHouseUpdateCount()
|
||||
// task.statHouseUpdateCount()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -99,37 +99,35 @@ class EsMonthHouseStat {
|
||||
}
|
||||
|
||||
def houseRun(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
|
||||
def city = d['B']
|
||||
if(city == "湖州"){
|
||||
def count = findHouseCount(city, 20210301, 20210601)
|
||||
d['E'] = count
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d->
|
||||
def city = d['A']
|
||||
def count = findHouseCount(city, 20190709, 20210709)
|
||||
d['E'] = count
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['D'] = precent+"%"
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['F'] = precent+"%"
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
+453
@@ -0,0 +1,453 @@
|
||||
package com.fangjia.script.gj.guangfa
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class GuangfaBlankSelectDistrict {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new GuangfaBlankSelectDistrict();
|
||||
// task.run()
|
||||
task.run2()
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def file = "D:\\数据\\广发\\获取楼盘信息需求(东莞和杭州).xlsx"
|
||||
def cityMap = standardRegion(file)
|
||||
def priceMap = getZhiDaoPrice(file)
|
||||
selectDistricts(cityMap, priceMap)
|
||||
println "写入完成"
|
||||
}
|
||||
|
||||
def run2(){
|
||||
selectDistricts2()
|
||||
println "写入完成"
|
||||
}
|
||||
|
||||
def selectDistricts(cityMap, priceMap){
|
||||
def rets = []
|
||||
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
|
||||
cityMap.each{city, regions->
|
||||
if(["西安","东莞","杭州"].contains(city)){
|
||||
def thirdPriceMap = findNearThirdPartyPrice(city, regions)
|
||||
def retMaps = findCreditHightDistrict2(city, regions, "2021-08-01", priceMap, thirdPriceMap)
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
|
||||
//基本情况合格的小区
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
if(regions.contains(d.region)){// && d.household_count && d.household_count>=50
|
||||
def distKeys = retMaps.get(d.region)
|
||||
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
|
||||
if(d.finish_building_date){
|
||||
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
|
||||
if(year){
|
||||
def mod = 2021-year
|
||||
if(mod <=15){
|
||||
d.house_year = mod
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
// if(d.finish_building_date){
|
||||
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
|
||||
// if(year){
|
||||
// def mod = 2021-year
|
||||
// if(mod <=15){
|
||||
// def distKeys = retMaps.get(d.region)
|
||||
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
|
||||
// d.house_year = mod
|
||||
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
println city +" 基本情况合格的小区:"+distMap.size()
|
||||
|
||||
for(def r : regions){
|
||||
def names = retMaps.get(r)
|
||||
if(names){
|
||||
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
|
||||
rets.addAll(results)
|
||||
}
|
||||
}
|
||||
println "结束"
|
||||
}
|
||||
}
|
||||
|
||||
println "开始写入Excel....."+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
def selectDistricts2(){
|
||||
def rets = []
|
||||
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
|
||||
def citys = ["东莞","杭州"]
|
||||
citys.each{city->
|
||||
def thirdPriceMap = findNearThirdPartyPrice(city)
|
||||
def retMaps = findCreditHightDistrict3(city, "2021-08-22", thirdPriceMap)
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
|
||||
//基本情况合格的小区
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
// && d.household_count && d.household_count>=50
|
||||
def distKeys = retMaps.get(d.region)
|
||||
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
|
||||
if(d.finish_building_date){
|
||||
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
|
||||
if(year){
|
||||
def mod = 2021-year
|
||||
if(mod <=15){
|
||||
d.house_year = mod
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
|
||||
}else{
|
||||
d.house_year = null
|
||||
}
|
||||
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
// if(d.finish_building_date){
|
||||
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
|
||||
// if(year){
|
||||
// def mod = 2021-year
|
||||
// if(mod <=15){
|
||||
// def distKeys = retMaps.get(d.region)
|
||||
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
|
||||
// d.house_year = mod
|
||||
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
println city +" 基本情况合格的小区:"+distMap.size()
|
||||
|
||||
for(def r : regions){
|
||||
def names = retMaps.get(r)
|
||||
if(names){
|
||||
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
|
||||
rets.addAll(results)
|
||||
}
|
||||
}
|
||||
println "结束"
|
||||
}
|
||||
|
||||
println "开始写入Excel....."+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(names.size())
|
||||
|
||||
for(def i=0;i<names.size();i++){
|
||||
def key = names[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def d = distMap.get(key)
|
||||
if(d){
|
||||
def ret = getDetailInfo(d, priceMap)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
// latch.await()
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
return rets
|
||||
}
|
||||
|
||||
def getDetailInfo(d, priceMap){
|
||||
def retMap1 = null;
|
||||
def retMap2 = null;
|
||||
def retMap3 = null
|
||||
if(d.city == "上海"){
|
||||
retMap1 = findShTrades(d, "2021-01-01", "2021-07-01")
|
||||
retMap2 = findShTrades(d, "2020-01-01", "2020-12-31")
|
||||
retMap3 = findShTrades(d, "2019-01-01", "2019-12-31")
|
||||
}else{
|
||||
retMap1 = CcbCommonUtils.findTrades(d, "2021-01-01", "2021-07-01")
|
||||
retMap2 = CcbCommonUtils.findTrades(d, "2020-01-01", "2020-12-31")
|
||||
retMap3 = CcbCommonUtils.findTrades(d, "2019-01-01", "2019-12-31")
|
||||
}
|
||||
if(retMap1){
|
||||
d.count1 = retMap1.count
|
||||
d.avg_price1 = (int)retMap1.avg_price
|
||||
}
|
||||
if(retMap2){
|
||||
d.count2 = retMap2.count
|
||||
d.avg_price2 = (int)retMap2.avg_price
|
||||
}
|
||||
if(retMap3){
|
||||
d.count3 = retMap3.count
|
||||
d.avg_price3 = (int)retMap3.avg_price
|
||||
}
|
||||
if(!retMap1 || !retMap2 || !retMap3){
|
||||
println d.city+" "+ d.region+" "+ d.name +"成交有一年为空过滤"
|
||||
return null
|
||||
}
|
||||
|
||||
def houses = EsSearchUtils.findSellHouseDetail(d.city, d.region, d.name)
|
||||
def rooms = []
|
||||
def areas = []
|
||||
for(def h : houses){
|
||||
rooms << h.room
|
||||
areas << h.area
|
||||
}
|
||||
rooms = rooms.unique().sort()
|
||||
areas = areas.unique().sort()
|
||||
if(rooms){
|
||||
d.room = rooms[0]+"-"+rooms[rooms.size()-1]
|
||||
d.area = areas[0]+"-"+areas[areas.size()-1]
|
||||
}
|
||||
|
||||
def zhidaoprice = priceMap.get(d.city+"#"+d.region)
|
||||
def ret = [d.city, d.region, zhidaoprice, d.name, d.address, d.developer, d.cubage_percent, d.estate_company, d.household_count, d.house_year,d.room, d.area, d.count1, d.avg_price1,d.count2, d.avg_price2,d.count3, d.avg_price3]
|
||||
println ret
|
||||
return ret;
|
||||
}
|
||||
|
||||
def standardRegion(filePath){
|
||||
def city = null
|
||||
def cityMap = new LinkedHashMap()
|
||||
def index = 0
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
index ++
|
||||
if(index >= 4){
|
||||
if(row['C']){
|
||||
city = row['C']
|
||||
}
|
||||
println city+" "+row['D']
|
||||
def newRegion = null
|
||||
if(city && row['D']){
|
||||
newRegion = RegionData.formatRegion(city, row['D'])
|
||||
row['F'] = newRegion
|
||||
}
|
||||
if(city && newRegion){
|
||||
def regions = cityMap.get(city)
|
||||
if(!regions){
|
||||
regions = []
|
||||
cityMap.put(city, regions)
|
||||
}
|
||||
if(!regions.contains(newRegion)){
|
||||
regions << newRegion
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return cityMap
|
||||
}
|
||||
|
||||
|
||||
def findCreditHightDistrict2(def city, def regions, def weekend, def priceMap, def thirdPriceMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
if(regions.contains(d.region)){
|
||||
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
|
||||
if(thirdPrice){
|
||||
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
|
||||
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
|
||||
def price = priceMap.get(d.city +"#"+d.region)
|
||||
if(d.avg_price>price){
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
rets << d.city+"##"+d.region+"##"+d.district_name
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
rets << d.city+"##"+d.region+"##"+d.district_name
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return retMaps;
|
||||
}
|
||||
|
||||
def getZhiDaoPrice(filePath){
|
||||
def priceMap = [:]
|
||||
def city = null
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if(row['C']){
|
||||
city = row['C']
|
||||
}
|
||||
if(city && row['F']){
|
||||
def key = city +"#"+row['F']
|
||||
if(!priceMap.containsKey(key)){
|
||||
priceMap.put(key, row['E'])
|
||||
}
|
||||
}
|
||||
})
|
||||
return priceMap
|
||||
}
|
||||
|
||||
public static def findShTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fjCity", dist.city)
|
||||
q.put("fjRegion", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("fjName", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("fjName", dist.name)
|
||||
}
|
||||
|
||||
q.put("contractTime", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = youda_esfhouse_coll.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.price){
|
||||
total_price = total_price + d.price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
|
||||
if(count && avg_price){
|
||||
def retMap = [:]
|
||||
retMap.count = count
|
||||
retMap.avg_price = avg_price
|
||||
return retMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findNearThirdPartyPrice(def city, regions){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
q.put("s_date", new BasicDBObject('$gte', 202105))
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def thirdPriceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.region+"#"+d.name
|
||||
if(!thirdPriceMap.containsKey(key)){
|
||||
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return thirdPriceMap
|
||||
}
|
||||
|
||||
|
||||
def findNearThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202105))
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def thirdPriceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.region+"#"+d.name
|
||||
if(!thirdPriceMap.containsKey(key)){
|
||||
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return thirdPriceMap
|
||||
}
|
||||
|
||||
def findCreditHightDistrict3(def city, def regions, def weekend, def thirdPriceMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
|
||||
if(thirdPrice){
|
||||
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
|
||||
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
rets << d.city+"##"+d.region+"##"+d.district_name
|
||||
}
|
||||
}else{
|
||||
def rets = retMaps.get(d.region)
|
||||
if(!rets){
|
||||
rets = []
|
||||
retMaps.put(d.region, rets)
|
||||
}
|
||||
rets << d.city+"##"+d.region+"##"+d.district_name
|
||||
}
|
||||
}
|
||||
return retMaps;
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class DistrictLocationUpdate {
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationUpdate()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
|
||||
|
||||
|
||||
updateLocation(city, dists);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def queryPoiMap(def city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and source = '¸ßµÂ'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def retMap = [:]
|
||||
while (rs.next()) {
|
||||
def map = [:]
|
||||
map.city = rs.getString("cityname")
|
||||
map.region = rs.getString("adname")
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.type = rs.getString("type")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
|
||||
def name = NameProcess.nameStandard(map.city, map.region, map.name)
|
||||
if(name){
|
||||
retMap.put(map.region+"##"+name, map)
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
}
|
||||
+198
-50
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
@@ -14,7 +15,13 @@ import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@@ -22,59 +29,81 @@ import java.util.regex.Pattern;
|
||||
class LocationTest {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
Date date=new Date(1622802609);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
String str = format.format(date);
|
||||
System.out.println(str);
|
||||
// Date date=new Date(1622802609);
|
||||
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
// String str = format.format(date);
|
||||
// System.out.println(str);
|
||||
|
||||
// def task = new LocationTest()
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def regionPointsMap = findRegionRange()
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def city = "杭州"
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def index = 0
|
||||
def rets = []
|
||||
for(def dist : districts){
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
if(!flag){
|
||||
index++
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
|
||||
def loactionMap = matchMap(dist)
|
||||
if(loactionMap){
|
||||
ret << loactionMap.source
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
ret << loactionMap.address
|
||||
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
ret << distance
|
||||
ret << loactionMap.lng
|
||||
ret << loactionMap.lat
|
||||
if(dist.region==loactionMap.region){
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
|
||||
}else{
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
if(!flag) {
|
||||
index++
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
|
||||
def loactionMap = matchMap(dist)
|
||||
if (loactionMap) {
|
||||
ret << loactionMap.source
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
ret << loactionMap.address
|
||||
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
ret << distance
|
||||
ret << loactionMap.lng
|
||||
ret << loactionMap.lat
|
||||
if (dist.region == loactionMap.region) {
|
||||
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
|
||||
} else {
|
||||
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
|
||||
}
|
||||
} else {
|
||||
ret << ""
|
||||
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,地图没找到"
|
||||
}
|
||||
if (ret) {
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
|
||||
}
|
||||
rets << ret
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内")
|
||||
latch.await()
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
@@ -133,10 +162,12 @@ class LocationTest {
|
||||
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d->
|
||||
def index = 0
|
||||
ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d->
|
||||
index++
|
||||
def distance = null
|
||||
if(d['G']){
|
||||
distance = d['G'].toDouble()
|
||||
if(d['H']){
|
||||
distance = d['H'].toDouble()
|
||||
}
|
||||
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
|
||||
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
|
||||
@@ -180,12 +211,59 @@ class LocationTest {
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
|
||||
def dbo = DbUtils.seaweed.findOne(q)
|
||||
d['K'] = "修正"
|
||||
}
|
||||
}else if(d['E'] && d['B'] != d['E'] && distance && distance<=1000){
|
||||
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
|
||||
if(mapName.contains("-") && mapName.endsWith("楼")){
|
||||
mapName = mapName.substring(0, mapName.indexOf("-"))
|
||||
}
|
||||
if(d['C']==mapName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['E'])
|
||||
q.put("name", d['C'])
|
||||
q.put("cat", 'district')
|
||||
q.put("status", 0)
|
||||
def dbo = DbUtils.seaweed.findOne(q)
|
||||
if(!dbo){
|
||||
q.put("region", d['B'])
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(d['D']=="高德"){
|
||||
lng = (d['I'].toDouble()*10000000000).longValue()
|
||||
lat = (d['J'].toDouble()*10000000000).longValue()
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
|
||||
if(gmap){
|
||||
lng2 = (gmap.get("lng")*10000000000).longValue()
|
||||
lat2 = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
|
||||
}else{
|
||||
lng2 = (d['I'].toDouble()*10000000000).longValue()
|
||||
lat2 = (d['J'].toDouble()*10000000000).longValue()
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
|
||||
if(gmap){
|
||||
lng = (gmap.get("lng")*10000000000).longValue()
|
||||
lat = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", d['E'])
|
||||
udata.put("lng", lng)
|
||||
udata.put("lat", lat)
|
||||
udata.put("lng2", lng2)
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
d['K'] = "修正区域"
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
},0)
|
||||
|
||||
}
|
||||
@@ -236,7 +314,7 @@ class LocationTest {
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
if(baiduRest){
|
||||
bdlocationMap = getMapLocation(dist, baiduRest);
|
||||
if(bdlocationMap){
|
||||
@@ -280,7 +358,7 @@ class LocationTest {
|
||||
region = r.district
|
||||
address = r.address
|
||||
|
||||
if(r.district.contains(dist.region)){
|
||||
if(r.district && r.district.contains(dist.region)){
|
||||
score = 10
|
||||
}else {
|
||||
score = 5
|
||||
@@ -303,7 +381,7 @@ class LocationTest {
|
||||
mapName = r.name
|
||||
address = r.address
|
||||
|
||||
if(r.district.contains(dist.region)){
|
||||
if(r.district && r.district.contains(dist.region)){
|
||||
score = 9
|
||||
}else {
|
||||
score = 5
|
||||
@@ -320,7 +398,7 @@ class LocationTest {
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
address = r.address
|
||||
if(r.district.contains(dist.region)){
|
||||
if(r.district && r.district.contains(dist.region)){
|
||||
score = 8
|
||||
}else {
|
||||
score = 5
|
||||
@@ -338,7 +416,7 @@ class LocationTest {
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
address = r.address
|
||||
if(r.district.contains(dist.region)){
|
||||
if(r.district && r.district.contains(dist.region)){
|
||||
score = 7
|
||||
}else {
|
||||
score = 5
|
||||
@@ -366,15 +444,85 @@ class LocationTest {
|
||||
return flag;
|
||||
}
|
||||
|
||||
def findRegionRange(){
|
||||
def findRegionRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州市")
|
||||
q.put("city",city+"市")
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion("福州",d.region)
|
||||
def region = RegionData.formatRegion(city,d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
|
||||
def run5(){
|
||||
def results = getAllErrorDistricts()
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
for(def i=0;i<results.size();i++){
|
||||
final def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = mapLocation(map)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误")
|
||||
}
|
||||
|
||||
def mapLocation(map){
|
||||
def dist = CommonDatautils.findDistrictById(map[1])
|
||||
if(dist){
|
||||
def loactionMap = matchMap(dist)
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
if (loactionMap) {
|
||||
ret << loactionMap.source
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
ret << loactionMap.address
|
||||
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
ret << distance
|
||||
ret << loactionMap.lng
|
||||
ret << loactionMap.lat
|
||||
if (dist.region == loactionMap.region) {
|
||||
println dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
|
||||
} else {
|
||||
println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getAllErrorDistricts(){
|
||||
def sql = "select cityid,commid from bigdata_community_distcheck where distname<>checkdist or checkdist is null";
|
||||
|
||||
def rets = []
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
rets << [r.cityid, r.commid]
|
||||
});
|
||||
return rets
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
class Point {
|
||||
Double lat;
|
||||
Double lng;
|
||||
|
||||
public Point(Double lng, Double lat){
|
||||
this.lat = lat
|
||||
this.lng = lng
|
||||
}
|
||||
|
||||
Double getLat() {
|
||||
return lat
|
||||
}
|
||||
|
||||
void setLat(Double lat) {
|
||||
this.lat = lat
|
||||
}
|
||||
|
||||
Double getLng() {
|
||||
return lng
|
||||
}
|
||||
|
||||
void setLng(Double lng) {
|
||||
this.lng = lng
|
||||
}
|
||||
}
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.awt.geom.Point2D
|
||||
|
||||
class PointUtil {
|
||||
|
||||
public static final DBCollection BUSINESS_RANGE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border", "mongo.db.fangjia.com")
|
||||
|
||||
/**
|
||||
* 判断当前位置是否在多边形区域内
|
||||
*
|
||||
* @param checkPoint 当前经纬度
|
||||
* @param areaAround 围栏范围
|
||||
* @return
|
||||
*/
|
||||
public static boolean checkIsInPolygon(Point checkPoint, List<Point> areaAround) {
|
||||
Point2D.Double point = new Point2D.Double(checkPoint.getLat(), checkPoint.getLng());
|
||||
List<Point2D.Double> pointList = new ArrayList<>();
|
||||
for (Point everyPoint : areaAround) {
|
||||
Point2D.Double polygonPoint = new Point2D.Double(everyPoint.getLat(), everyPoint.getLng());
|
||||
pointList.add(polygonPoint);
|
||||
}
|
||||
return checkIsPtInPoly(point, pointList);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断点是否在多边形内,如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
|
||||
*
|
||||
* @param point 检测点
|
||||
* @param pts 多边形的顶点
|
||||
* @return 点在多边形内返回true, 否则返回false
|
||||
*/
|
||||
public static boolean checkIsPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
|
||||
int N = pts.size();
|
||||
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
|
||||
boolean boundOrVertex = true;
|
||||
//cross points count of x
|
||||
int intersectCount = 0;
|
||||
//浮点类型计算时候与0比较时候的容差
|
||||
double precision = 2e-10;
|
||||
//neighbour bound vertices
|
||||
Point2D.Double p1, p2;
|
||||
//当前点
|
||||
Point2D.Double p = point;
|
||||
//left vertex
|
||||
p1 = pts.get(0);
|
||||
//check all rays
|
||||
for (int i = 1; i <= N; ++i) {
|
||||
if (p.equals(p1)) {
|
||||
//p is an vertex
|
||||
return boundOrVertex;
|
||||
}
|
||||
//right vertex
|
||||
p2 = pts.get(i % N);
|
||||
//ray is outside of our interests
|
||||
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
|
||||
p1 = p2;
|
||||
//next ray left point
|
||||
continue;
|
||||
}
|
||||
//ray is crossing over by the algorithm (common part of)
|
||||
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
|
||||
//x is before of ray
|
||||
if (p.y <= Math.max(p1.y, p2.y)) {
|
||||
//overlies on a horizontal ray
|
||||
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
|
||||
return boundOrVertex;
|
||||
}
|
||||
//ray is vertical
|
||||
if (p1.y == p2.y) {
|
||||
//overlies on a vertical ray
|
||||
if (p1.y == p.y) {
|
||||
return boundOrVertex;
|
||||
//before ray
|
||||
} else {
|
||||
++intersectCount;
|
||||
}
|
||||
} else {
|
||||
//cross point on the left side
|
||||
//cross point of y
|
||||
double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
|
||||
//overlies on a ray
|
||||
if (Math.abs(p.y - xinters) < precision) {
|
||||
return boundOrVertex;
|
||||
}
|
||||
//before ray
|
||||
if (p.y < xinters) {
|
||||
++intersectCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//special case when ray is crossing through the vertex
|
||||
//p crossing over p2
|
||||
if (p.x == p2.x && p.y <= p2.y) {
|
||||
//next vertex
|
||||
Point2D.Double p3 = pts.get((i + 1) % N);
|
||||
//p.x lies between p1.x & p3.x
|
||||
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
|
||||
++intersectCount;
|
||||
} else {
|
||||
intersectCount += 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
//next ray left point
|
||||
p1 = p2;
|
||||
}
|
||||
//偶数在多边形外
|
||||
if (intersectCount % 2 == 0) {
|
||||
return false;
|
||||
} else {
|
||||
//奇数在多边形内
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(def a){
|
||||
new PointUtil().run()
|
||||
}
|
||||
def run(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", "淞南")
|
||||
def obj = BUSINESS_RANGE_COLL.findOne(q)
|
||||
if(obj){
|
||||
def pointStr = obj.border
|
||||
if(pointStr){
|
||||
def pointArrs = pointStr.split(";")
|
||||
def list = []
|
||||
for(def s : pointArrs){
|
||||
def arr = s.split(",")
|
||||
list << new Point(arr[0].toDouble(), arr[1].toDouble())
|
||||
}
|
||||
|
||||
println checkIsInPolygon(new Point(121.491478,31.327947), list)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -521,6 +521,10 @@ class GaoDeUtils {
|
||||
return KeyList.get(count);
|
||||
}
|
||||
|
||||
static void main(def a){
|
||||
println search("如东县大豫镇豫东村委会","南通")
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+20
-16
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.price
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -11,32 +12,35 @@ class EvaluationBatch {
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\农行\\东莞农行测试数据(御景花园18条估值)(人工询价).xlsx"
|
||||
def filePath = "D:\\数据\\银联\\房价网测试数据.xlsx"
|
||||
def city = "¶«Ý¸"
|
||||
ExcelUtils.modify(filePath, { data ->
|
||||
|
||||
if(!data['M']){
|
||||
// def address = data['C'].replace("\"","").trim()
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(city, data['K'],data['L'], address );
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, data['D'],data['E'], null);
|
||||
if(rMap){
|
||||
def json = evaluation(rMap, data['H'], 7, data['I'],null)
|
||||
def region = RegionData.formatRegion(data['A'], data['B'])
|
||||
if(region){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(data['A'], region, data['C'], null);
|
||||
if(rMap){
|
||||
def json = evaluation(rMap, data['D'], null, null,null)
|
||||
|
||||
// data['J'] = rMap.city
|
||||
// data['K'] = rMap.region
|
||||
// data['L'] = rMap.name
|
||||
if(json){
|
||||
def avgPrice = json.avg_price
|
||||
def totalPrice = json.total_price
|
||||
|
||||
data['N'] = totalPrice
|
||||
data['M'] = avgPrice
|
||||
data['O'] = json.credit
|
||||
data['P'] = rMap.region
|
||||
data['Q'] = rMap.name
|
||||
data['E'] = rMap.city
|
||||
data['F'] = rMap.region
|
||||
data['G'] = rMap.name
|
||||
if(json){
|
||||
def avgPrice = json.avg_price
|
||||
def totalPrice = json.total_price
|
||||
|
||||
data['H'] = totalPrice
|
||||
data['I'] = avgPrice
|
||||
data['J'] = json.credit
|
||||
// data['P'] = rMap.region
|
||||
// data['Q'] = rMap.name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}, 0);
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package com.fangjia.script.gj.price
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class NoPriceCalcutePrice {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new NoPriceCalcutePrice()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
|
||||
def map = getDistrictByCity(city, "2021-08-01")
|
||||
for(def d : districts){
|
||||
if(!map.containsKey(d.region+"@"+d.name)){
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(d.city, d.region, d.name, 12);
|
||||
println d.city+" "+d.region+" "+d.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getDistrictByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.fangjia.script.gj.region
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class MapAreaUpdate {
|
||||
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new MapAreaUpdate()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findAllCity()
|
||||
for(def city : rets){
|
||||
def retMap = findCityAndProvCode(city)
|
||||
if(retMap){
|
||||
update(retMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def update(retMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city",retMap.fangjia_city)
|
||||
q.put("district", "ÊÐÏ½Çø")
|
||||
def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city_code", retMap.city_code)
|
||||
udata.put("province_code", retMap.province_code)
|
||||
FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false)
|
||||
}
|
||||
}
|
||||
|
||||
def findCityAndProvCode(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city",city)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.city_code && d.province_code && d.district_code){
|
||||
map.put("fangjia_city",d.fangjia_city)
|
||||
map.put("city_code",d.city_code)
|
||||
map.put("province_code",d.province_code)
|
||||
break
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def findAllCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_district",null)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def citys = []
|
||||
for(def d : dbcursor){
|
||||
citys << d.fangjia_city
|
||||
}
|
||||
citys.unique()
|
||||
return citys
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -90,7 +90,7 @@ class AddressStandard {
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
},0, "Sheet1")
|
||||
|
||||
}
|
||||
|
||||
|
||||
+6
-6
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","¸£ÖÝ")
|
||||
q.put("source", "°²¾Ó¿Í")
|
||||
q.put("city","福州")
|
||||
q.put("source", "城市房产")
|
||||
q.put("s_date",202012)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = "anjuke.com/"
|
||||
def prefix = ".cityhouse.cn/ha/"
|
||||
def keyList =new HashSet()
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
@@ -77,11 +77,11 @@ class CrawlDistrictIdSelect {
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com"))
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
@@ -92,7 +92,7 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "anjuke.com"
|
||||
map.root = "cityhouse.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class MatchDistrict {
|
||||
static main(args) {
|
||||
def task = new MatchDistrict()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx"
|
||||
def city = "深圳"
|
||||
ExcelUtils.multiThreadingModify(filePath, { data ->
|
||||
if(data['A']){
|
||||
def region = RegionData.formatRegion(city, data['A'])
|
||||
if(region){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null);
|
||||
if(rMap){
|
||||
data['E'] = rMap.city
|
||||
data['F'] = rMap.region
|
||||
data['G'] = rMap.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0);
|
||||
}
|
||||
}
|
||||
|
||||
//{
|
||||
// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"),
|
||||
// "guiding_type" : NumberInt(0),
|
||||
// "estate_type" : "普通住宅",
|
||||
// "house_type" : NumberInt(0),
|
||||
// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"),
|
||||
// "guiding_person" : "深圳房管局",
|
||||
// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"),
|
||||
// "category" : "district",
|
||||
// "guiding_value" : NumberInt(48300),
|
||||
// "name" : "中粮云景国际",
|
||||
// "city" : "深圳",
|
||||
// "region" : "光明"
|
||||
//}
|
||||
@@ -1,7 +1,57 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
Date date = new Date(1617211269868);
|
||||
print dateFormat.format(date);
|
||||
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
//Date date = new Date(1617211269868);
|
||||
//print dateFormat.format(date);
|
||||
//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
|
||||
//def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
|
||||
//ExcelUtils.modify(path,{d->
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
|
||||
// d['G'] = districts.size()
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city", d['B'])
|
||||
// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
|
||||
// def obj = CITY_INFO_CONN.findOne(q);
|
||||
// if(obj){
|
||||
// d['H'] = obj.price_coverage*100
|
||||
// }
|
||||
//})
|
||||
|
||||
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
|
||||
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
|
||||
def countMap = [:]
|
||||
onLineSqlInstance.eachRow(sql,{r->
|
||||
def addTime = r.visit_time
|
||||
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
|
||||
def ips = countMap.get(month)
|
||||
if(!ips){
|
||||
ips = new HashSet();
|
||||
countMap.put(month, ips)
|
||||
}
|
||||
ips << r.ip
|
||||
});
|
||||
|
||||
countMap.each {k,v->
|
||||
println k+" "+v.size()
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+11
-1
@@ -47,12 +47,22 @@ class SFTP {
|
||||
* @param sftp
|
||||
*/
|
||||
public void upload(String directory, String uploadFile, ChannelSftp sftp) {
|
||||
FileInputStream stream = null
|
||||
try {
|
||||
sftp.cd(directory);
|
||||
File file=new File(uploadFile);
|
||||
sftp.put(new FileInputStream(file), file.getName());
|
||||
stream = new FileInputStream(file)
|
||||
sftp.put(stream, file.getName());
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if(stream!=null){
|
||||
stream.close();
|
||||
}
|
||||
// if (sftp != null) {
|
||||
// sftp.disconnect();
|
||||
// sftp.exit();
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+13
-9
@@ -35,17 +35,21 @@ class InsidePolygon {
|
||||
|
||||
|
||||
public static List<Point> createPoint2(String points){
|
||||
String[] array = points.split(";");
|
||||
|
||||
List<Point> list = new ArrayList<Point>();
|
||||
for(int i =0 ;i<array.length-1;i++){
|
||||
def pointArrs = array[i].split("[,|\\|]")
|
||||
Point point = new Point();
|
||||
double x = Double.parseDouble(pointArrs[0]);
|
||||
double y = Double.parseDouble(pointArrs[1]);
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
if(points){
|
||||
String[] array = points.split(";");
|
||||
for(int i =0 ;i<array.length-1;i++){
|
||||
def pointArrs = array[i].split("[,|\\|]")
|
||||
Point point = new Point();
|
||||
double x = Double.parseDouble(pointArrs[0]);
|
||||
double y = Double.parseDouble(pointArrs[1]);
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
|
||||
/**
|
||||
* 快照监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/8/27
|
||||
*/
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SnapshotMonitor extends BaseBatch {
|
||||
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SnapshotMonitor(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// /_snapshot/es_backup/weekly-snap-2021.05*
|
||||
// 监控两个月的快照
|
||||
String[] months = [DateUtils.formatDateByPattern(new Date(), "yyyy.MM"), DateUtils.formatDateByPattern(DateUtil.addMonthToDate(new Date(), -1), "yyyy.MM")]
|
||||
String urlPrefix = "http://192.168.0.206:9200/_snapshot/es_backup/weekly-snap-"
|
||||
|
||||
String msg = "";
|
||||
for (String month : months) {
|
||||
// 必须分开写,不能有空格
|
||||
String[] cmds = ["curl", "-XGET", urlPrefix + month + "*", "--user", "elastic:goojia123456"]
|
||||
String result = execCurl(cmds)
|
||||
JSONObject jsonObject = JSONObject.parse(result);
|
||||
|
||||
if (jsonObject.get("snapshots") != null) {
|
||||
com.alibaba.fastjson.JSONArray jsonArray = jsonObject.get("snapshots")
|
||||
for (JSONObject obj : jsonArray) {
|
||||
String state = obj.get("state")
|
||||
if (!"SUCCESS".equals(state)) {
|
||||
msg = "【ES集群快照异常】\n" + obj.get("failures").toString()
|
||||
DingDingMessageUtil.sendTextMessage(access_token, msg)
|
||||
return ;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static String execCurl(String[] cmds) {
|
||||
ProcessBuilder process = new ProcessBuilder(cmds);
|
||||
Process p;
|
||||
try {
|
||||
p = process.start();
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
|
||||
StringBuilder builder = new StringBuilder();
|
||||
String line = null;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
builder.append(line);
|
||||
builder.append(System.getProperty("line.separator"));
|
||||
}
|
||||
return builder.toString();
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.fangjia.script.test
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class RegionalismCodeStat {
|
||||
|
||||
DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionalismCodeStat()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def provinces = ["上海市","北京市","天津市","重庆市","福建省","贵州省","湖南省","广东省","湖北省","安徽省","四川省","江苏省","吉林省","河北省","河南省","海南省","江西省","云南省","甘肃省","山东省","陕西省","浙江省","青海省","辽宁省","黑龙江省","山西省","西藏自治区","新疆维吾尔自治区","宁夏回族自治区","内蒙古自治区","广西壮族自治区"]
|
||||
// def provinces = ["宁夏回族自治区"]
|
||||
provinces.each {v->
|
||||
def sets = findCount(v)
|
||||
println v +" "+ sets.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findCount(v){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", v)
|
||||
def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000)
|
||||
def set = new HashSet()
|
||||
for(def obj : dbcursor){
|
||||
// if(obj.city=="省直辖县级行政区划" || obj.city=="自治区直辖县级行政区划"){
|
||||
// //set << obj.region
|
||||
// println "【"+obj.region+"】 过滤"
|
||||
// continue
|
||||
// }else{
|
||||
// if(obj.region && !set.contains(obj.region)){
|
||||
// set << obj.region
|
||||
// println "【"+obj.region+"】 add"
|
||||
// }
|
||||
// }
|
||||
|
||||
if(obj.juweihui){
|
||||
def k = obj.street+"##"+obj.juweihui
|
||||
if(!set.contains(k)){
|
||||
set << k
|
||||
// println "【"+k+"】 add"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// set.each {k->
|
||||
// println k
|
||||
// }
|
||||
return set;
|
||||
}
|
||||
}
|
||||
+149
-41
@@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.CommonUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
@@ -59,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
private static def limit = 5000;
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
@@ -81,7 +83,6 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def flag = true;
|
||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||
|
||||
@@ -120,26 +121,35 @@ class DataStandardBatch extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||
// if(dbo.avg_price){
|
||||
if(dbo.name){
|
||||
|
||||
//标准化城市(包含region)
|
||||
def data = standardCity(dbo);
|
||||
if(data){
|
||||
//匹配区域
|
||||
data = standardRegion(data);
|
||||
|
||||
if(data){
|
||||
//标准化小区名
|
||||
def data3 = standardDistrict(data, dbo.avg_price);
|
||||
if(data.city=="三亚" && data.region=="陵水"){
|
||||
data.city = "陵水"
|
||||
data.region = ""
|
||||
def data3 = standardDistrictSimple(data, dbo.avg_price);
|
||||
if(data3){
|
||||
successCount++;
|
||||
}
|
||||
}else if(data){
|
||||
data = standardRegion(data);
|
||||
if(data){
|
||||
//标准化小区名
|
||||
def data3 = standardDistrict(data, dbo.avg_price);
|
||||
if(data3){
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
// }else{
|
||||
// saveLog(dbo, "无价格");
|
||||
// updateStatus(dbo,statusMap.get("无价格"))
|
||||
// }
|
||||
}else{
|
||||
saveLog(dbo, "无名称");
|
||||
updateStatus(dbo,statusMap.get("无名称"))
|
||||
}
|
||||
|
||||
count++;
|
||||
}catch (Exception e){
|
||||
@@ -253,6 +263,104 @@ class DataStandardBatch extends BaseBatch{
|
||||
return data;
|
||||
}
|
||||
|
||||
def standardDistrict2(def data, def avg_price){
|
||||
def newName = NameProcess.nameStandard(data.city, data.region, data.name)
|
||||
def district = null
|
||||
if(newName){
|
||||
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address);
|
||||
if(district){
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(district && avg_price){
|
||||
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
|
||||
saveSuccessLog(data, district, type);
|
||||
updateStatus(data,statusMap.get("小区匹配成功"))
|
||||
println district.city+" "+district.name+" success"
|
||||
}else{
|
||||
if(district){
|
||||
// data.type = type
|
||||
saveLog(data, "无价格");
|
||||
updateStatus(data,statusMap.get("无价格"))
|
||||
println data.city+" "+data.name+" fail(无价格)"
|
||||
}else{
|
||||
println "[匹配失败]"
|
||||
saveLog(data, "小区匹配失败");
|
||||
updateStatus(data,statusMap.get("小区匹配失败"))
|
||||
println data.city+" "+data.name+" fail"
|
||||
}
|
||||
}
|
||||
return district;
|
||||
|
||||
}
|
||||
|
||||
def standardDistrictSimple(def data, def avg_price){
|
||||
def newName1 = null
|
||||
def newName2 = null
|
||||
if(data.name.contains("别名")){
|
||||
newName1 = data.name.toString().substring(0, data.name.toString().indexOf("别名")-1)
|
||||
newName2 = data.name.toString().substring(data.name.toString().indexOf("别名")+3)
|
||||
}else{
|
||||
newName1 = NameProcess.nameStandard(data.city, data.region, data.name)
|
||||
}
|
||||
def district = null
|
||||
def type = null
|
||||
if(newName1){
|
||||
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address);
|
||||
if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){
|
||||
district._id = data._id
|
||||
type = 1
|
||||
}else{
|
||||
if(newName2){
|
||||
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address);
|
||||
if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){
|
||||
district._id = data._id
|
||||
type = 1
|
||||
}else{
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2)
|
||||
if(score>0.8){
|
||||
district._id = data._id
|
||||
type = 11
|
||||
}else{
|
||||
district = null
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(district){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1)
|
||||
if(score>0.8){
|
||||
district._id = data._id
|
||||
type = 11
|
||||
}else{
|
||||
district = null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(district && avg_price){
|
||||
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
|
||||
saveSuccessLog(data, district, type);
|
||||
updateStatus(data,statusMap.get("小区匹配成功"))
|
||||
println district.city+" "+district.name+" success"
|
||||
}else{
|
||||
if(district){
|
||||
data.type = type
|
||||
saveLog(data, "无价格");
|
||||
updateStatus(data,statusMap.get("无价格"))
|
||||
println data.city+" "+data.name+" fail(无价格)"
|
||||
}else{
|
||||
println "[匹配失败]"
|
||||
saveLog(data, "小区匹配失败");
|
||||
updateStatus(data,statusMap.get("小区匹配失败"))
|
||||
println data.city+" "+data.name+" fail"
|
||||
}
|
||||
}
|
||||
return district;
|
||||
}
|
||||
|
||||
def standardDistrict(def data, def avg_price){
|
||||
//名称匹配,名称别名匹配,地址别名匹配
|
||||
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
|
||||
@@ -368,30 +476,10 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
if(district){
|
||||
if(data.region != district.region){
|
||||
//type = 12;//区域不同
|
||||
type = 12;//区域不同
|
||||
district = null;
|
||||
}else{
|
||||
def score1 = 0;
|
||||
if(data.address && district.address){
|
||||
score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address);
|
||||
}
|
||||
if(score1 && score1 >=0.8){
|
||||
type = 13;//地址相似度在0.8以上
|
||||
}else{
|
||||
if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){
|
||||
type = 13;
|
||||
}else{
|
||||
if(!addr){
|
||||
def openDistName = district.name.replace("小区","")
|
||||
def name = data.name.replace("小区","")
|
||||
if(openDistName.contains(name) || name.contains(openDistName)){
|
||||
type = 15;//匹配接口通过名称配上,名称包含
|
||||
}
|
||||
}
|
||||
}
|
||||
//type = 11;
|
||||
district = null;
|
||||
}
|
||||
type=15
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -495,10 +583,16 @@ class DataStandardBatch extends BaseBatch{
|
||||
}*/
|
||||
|
||||
|
||||
def getMatchNameFromMap(def city, def region, def disName, def address){
|
||||
def getMatchNameFromMap(def city, def region, def disName, String address){
|
||||
//查询房价网是否存在该小区
|
||||
def dbo = findSameNameFromMap(city, region, disName);
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
def addrFlag = null
|
||||
if(address){
|
||||
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(addrFlag){
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
}
|
||||
}
|
||||
if(!dbo){
|
||||
//别名
|
||||
def distrs = findAliasNameFromMap(city, region, disName);
|
||||
@@ -506,7 +600,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
dbo = distrs[0]
|
||||
}else if(distrs && distrs.size()>1 && address){
|
||||
for(def d : distrs){
|
||||
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
|
||||
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
|
||||
dbo = d;
|
||||
break;
|
||||
}
|
||||
@@ -570,10 +664,16 @@ class DataStandardBatch extends BaseBatch{
|
||||
return dbo;
|
||||
}*/
|
||||
def getMatchAddressFromMap(def city, def region, def address, def name){
|
||||
def dbo = findSameNameFromMap(city, region, address);
|
||||
def dbo = null
|
||||
if(address && isStandardRoad(address)){
|
||||
dbo = findSameNameFromMap(city, region, address);
|
||||
}
|
||||
if(!dbo){
|
||||
def result = findAddressFromMap(city, region, address);
|
||||
if(result.size()==1){
|
||||
def result = null
|
||||
if(address && isStandardRoad(address)){
|
||||
result = findAddressFromMap(city, region, address);
|
||||
}
|
||||
if(result && result.size()==1){
|
||||
dbo = result[0];
|
||||
}else if(result && result.size()>1 && name){
|
||||
def score = null;
|
||||
@@ -621,6 +721,15 @@ class DataStandardBatch extends BaseBatch{
|
||||
return dbo;
|
||||
}
|
||||
|
||||
def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
def addressCheck(def dbo, def region, def address){
|
||||
if(dbo && !region && address && address.length()>3){
|
||||
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){
|
||||
@@ -673,7 +782,6 @@ class DataStandardBatch extends BaseBatch{
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
// qdata.put("name", "银河湾花园(别墅)")
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
def ret = []
|
||||
try{
|
||||
|
||||
+164
-7
@@ -38,6 +38,152 @@ class Esf58Crawl extends BaseBatch{
|
||||
def static useDistKeys = []
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
def cityList = [
|
||||
"合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319",
|
||||
"安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","淮北":"huaibei|9357","滁州":"chuzhou|10266",
|
||||
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
|
||||
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
|
||||
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
|
||||
"福州":"fz|304","厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710",
|
||||
"宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761",
|
||||
"石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757",
|
||||
"福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133",
|
||||
"漳浦":"zhangpu|717",
|
||||
"深圳":"sz|4","广州":"gz|3","东莞":"dg|413","佛山":"fs|222","中山":"zs|771",
|
||||
"珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791",
|
||||
"肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303",
|
||||
"阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449",
|
||||
"潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725",
|
||||
"博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456",
|
||||
"南宁":"nn|845","柳州":"liuzhou|7133","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046",
|
||||
"北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340",
|
||||
"来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774",
|
||||
"北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119",
|
||||
"贵阳":"gy|2015","遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506",
|
||||
"毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628",
|
||||
"清镇":"qingzhen|12703",
|
||||
"兰州":"lz|952","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154",
|
||||
"酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428",
|
||||
"陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390",
|
||||
"海口":"haikou|2053","三亚":"sanya|2422","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
|
||||
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
|
||||
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380",
|
||||
"儋州":"danzhou|10394",
|
||||
"郑州":"zz|342","洛阳":"luoyang|556","新乡":"xx|1016","南阳":"ny|592","许昌":"xc|977",
|
||||
"平顶山":"pds|1005","安阳":"ay|1096","焦作":"jiaozuo|3266","商丘":"sq|1029","开封":"kaifeng|2342",
|
||||
"濮阳":"puyang|2346","周口":"zk|933","信阳":"xy|8694","驻马店":"zmd|1067","漯河":"luohe|2347",
|
||||
"三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123",
|
||||
"禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010",
|
||||
"项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101",
|
||||
"沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393",
|
||||
"通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219",
|
||||
"舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322",
|
||||
"沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185",
|
||||
"范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603",
|
||||
"哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
|
||||
"佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862",
|
||||
"伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721",
|
||||
"肇州":"zhaozhou|382",
|
||||
"武汉":"wh|158","宜昌":"yc|858","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
|
||||
"黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296",
|
||||
"咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517",
|
||||
"仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896",
|
||||
"武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484",
|
||||
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
|
||||
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
|
||||
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
|
||||
"长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914",
|
||||
"湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
|
||||
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
|
||||
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
|
||||
"祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473",
|
||||
"邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532",
|
||||
"湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095",
|
||||
"石家庄":"sjz|241","保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572",
|
||||
"秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993","张家口":"zjk|3328",
|
||||
"承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048",
|
||||
"正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577",
|
||||
"雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657",
|
||||
"沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395",
|
||||
"固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066",
|
||||
"沙河":"shaheshi|755",
|
||||
"苏州":"su|5","南京":"nj|172","无锡":"wx|93","常州":"cz|463","徐州":"xz|471",
|
||||
"南通":"nt|394","扬州":"yz|637","盐城":"yancheng|613","淮安":"ha|968","连云港":"lyg|2049",
|
||||
"泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
|
||||
"如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147",
|
||||
"扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402",
|
||||
"邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401",
|
||||
"东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148",
|
||||
"姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959",
|
||||
"句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976",
|
||||
"金湖":"jinhu|975","江阴":"jiangyins|34984",
|
||||
"南昌":"nc|669","赣州":"ganzhou|2363","九江":"jj|2247","宜春":"yichun|5709","吉安":"ja|2364",
|
||||
"上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115",
|
||||
"鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118",
|
||||
"丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137",
|
||||
"浮梁":"fuliangxian|9071",
|
||||
"长春":"cc|319","吉林":"jl|700","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315",
|
||||
"白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171",
|
||||
"梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708",
|
||||
"梨树县":"lishu|10176",
|
||||
"沈阳":"sy|188","大连":"dl|147","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722",
|
||||
"营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038",
|
||||
"本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306",
|
||||
"瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733",
|
||||
"银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421",
|
||||
"呼和浩特":"hu|811","包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015",
|
||||
"呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976",
|
||||
"乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043",
|
||||
"西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
|
||||
"黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904",
|
||||
"青岛":"qd|122","济南":"jn|265","烟台":"yt|228","潍坊":"wf|362","临沂":"linyi|505",
|
||||
"淄博":"zb|385","济宁":"jining|450","泰安":"ta|686","聊城":"lc|882","威海":"weihai|518",
|
||||
"枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","东营":"dy|623","菏泽":"heze|5632",
|
||||
"滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146",
|
||||
"寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690",
|
||||
"高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235",
|
||||
"蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967",
|
||||
"新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884",
|
||||
"茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640",
|
||||
"无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739",
|
||||
"乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888",
|
||||
"阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334",
|
||||
"汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238",
|
||||
"宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373",
|
||||
"临朐":"linqu|374","鄄城":"juancheng|5635",
|
||||
"太原":"ty|740","临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854",
|
||||
"长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453",
|
||||
"朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354",
|
||||
"泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227",
|
||||
"西安":"xa|483","咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163",
|
||||
"榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832",
|
||||
"神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948",
|
||||
"成都":"cd|102","绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380",
|
||||
"自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928",
|
||||
"遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803",
|
||||
"凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817",
|
||||
"甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706",
|
||||
"射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148",
|
||||
"乌鲁木齐":"xj|984","昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499",
|
||||
"喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475",
|
||||
"和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562",
|
||||
"图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845",
|
||||
"拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648",
|
||||
"那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684",
|
||||
"昆明":"km|541","曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040",
|
||||
"丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409",
|
||||
"德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432",
|
||||
"怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533",
|
||||
"杭州":"hz|79","宁波":"nb|135","温州":"wz|330","金华":"jh|531","嘉兴":"jx|497",
|
||||
"台州":"tz|403","绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793",
|
||||
"舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333",
|
||||
"诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334",
|
||||
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
|
||||
"安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409",
|
||||
"平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
|
||||
"江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new Esf58Crawl(), args)
|
||||
}
|
||||
@@ -45,18 +191,29 @@ class Esf58Crawl extends BaseBatch{
|
||||
void process(Object args) {
|
||||
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
for(def i=0;i<5;i++){
|
||||
def city = "¸£ÖÝ"
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
distMap.put(city, districts)
|
||||
def citys = ["成都","南京","福州","厦门","泉州","东莞","广州","广州","广州","西安","沈阳","大连","昆明","太原","郑州","乌鲁木齐","烟台","绵阳","邯郸","潍坊","嘉兴","惠州","珠海","海口"]
|
||||
cityList.each{city, str->
|
||||
for(def i=0;i<1;i++){
|
||||
// def city = "福州"
|
||||
if(citys.contains(city)){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
distMap.put(city, districts)
|
||||
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://fz.58.com/xiaoqu/");
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, url);
|
||||
|
||||
Thread.sleep(1000*60*10)
|
||||
// Thread.sleep(1000*60*10)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addCrawlKeys(city){
|
||||
def districts = distMap.get(city);
|
||||
for(def i=0;i<districts.size();i++){
|
||||
|
||||
+30
-11
@@ -1,8 +1,10 @@
|
||||
package com.fangjia.script.thirdPriceCrawl
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.solr.common.util.Hash
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* ¸£ÖÝÅ©ÉÌÐÐ
|
||||
@@ -13,8 +15,9 @@ class SynchDistrictPrice {
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
"anjuke.com":"安居客"]
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -29,16 +32,19 @@ class SynchDistrictPrice {
|
||||
def rets = findCrawlDistrictDetails(city, k, v)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, v)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findCrawlDistrictDetails(city, root, source){
|
||||
|
||||
def curMonthFirstDay = DateUtils.getFirstDayOfMonth(new Date())
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("root", root)
|
||||
// q.put("name","兰州万达城")
|
||||
q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay));
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
@@ -47,6 +53,7 @@ class SynchDistrictPrice {
|
||||
keys.put("address", 1)
|
||||
keys.put("avg_price", 1)
|
||||
keys.put("root", 1)
|
||||
keys.put("c_date", 1)
|
||||
keys.put("url", 1)
|
||||
def rets = []
|
||||
def dbcursor = districtDetailColl.find(q, keys)
|
||||
@@ -74,6 +81,7 @@ class SynchDistrictPrice {
|
||||
q.put("city", city)
|
||||
def count = thirdPartyColl.count(q)
|
||||
def addIndex = 0
|
||||
def updateIndex = 0
|
||||
if(count==0){
|
||||
try{
|
||||
thirdPartyColl.insert(rets)
|
||||
@@ -83,13 +91,27 @@ class SynchDistrictPrice {
|
||||
println "["+source +"-"+city + "] add:"+ rets.size()+" Ôץȡ£º"+count
|
||||
}else{
|
||||
for(def r : rets){
|
||||
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(!flag){
|
||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(obj){
|
||||
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}else{
|
||||
thirdPartyColl.save(r)
|
||||
}
|
||||
// if(!obj){
|
||||
// addIndex++
|
||||
// thirdPartyColl.save(r)
|
||||
// }else{
|
||||
// if(r.avg_price && !obj.avg_price){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("avg_price", r.avg_price)
|
||||
// thirdPartyColl.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',udata))
|
||||
// updateIndex++
|
||||
// }
|
||||
// }
|
||||
}
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
|
||||
}
|
||||
|
||||
|
||||
@@ -102,10 +124,7 @@ class SynchDistrictPrice {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
def count = thirdPartyColl.count(q)
|
||||
if(count){
|
||||
return true
|
||||
}
|
||||
return false
|
||||
def obj = thirdPartyColl.findOne(q)
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202105);
|
||||
def count2 = findTotalCount(source, 202106);
|
||||
def count1 = findTotalCount(source, 202108);
|
||||
def count2 = findTotalCount(source, 202109);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -41,8 +41,10 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
println SHORT_DATE
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
|
||||
def cityIndex = 0;
|
||||
final CountDownLatch latch = new CountDownLatch(citys.size())
|
||||
citys.each { city ->
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""])
|
||||
citys.addAll(["¸£ÖÝ"])
|
||||
// getNoDataCitys();
|
||||
def cityurl = "http://www.zhuge.com/areachange/"
|
||||
def cityUrls = getCityUrlCrawal(cityurl);
|
||||
|
||||
@@ -531,8 +531,10 @@ public class DataUtil {
|
||||
if(!(address ==~ /.*(½»»ã|½»¿Ú|½»²æ¿Ú)$/)){
|
||||
address += "½»»ã"
|
||||
}
|
||||
}else{
|
||||
address = null
|
||||
}
|
||||
address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
|
||||
// address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
|
||||
}
|
||||
return address
|
||||
}
|
||||
|
||||
@@ -921,13 +921,13 @@ class DateUtils {
|
||||
|
||||
public static String getRandomDate(def startDate, def endDate){
|
||||
Date randomDate = randomDate(startDate, endDate);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
|
||||
String result = format.format(randomDate);
|
||||
return result;
|
||||
}
|
||||
|
||||
static main(args){
|
||||
|
||||
println getRandomDate("2021-08-01","2021-08-31")
|
||||
// nowFirstDay();
|
||||
// nowLastDay();
|
||||
/*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
|
||||
@@ -955,8 +955,9 @@ class DateUtils {
|
||||
// Date randomDate = randomDate("2010-09-20", "2011-09-22");
|
||||
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
|
||||
// String result = format.format(randomDate);
|
||||
System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
|
||||
// System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
|
||||
|
||||
println DateUtils.getDayOfWeek(new Date())
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ class ElasticUtils {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
|
||||
@@ -60,7 +60,8 @@ public class ExcelUtils{
|
||||
FileInputStream inp = null;
|
||||
try {
|
||||
inp = new FileInputStream(file);
|
||||
Workbook workbook = WorkbookFactory.create(inp)
|
||||
BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 200)
|
||||
Workbook workbook = WorkbookFactory.create(bufferedInputStream)
|
||||
workbook.setForceFormulaRecalculation(true)
|
||||
Sheet sheet = workbook.getSheetAt(sheetIndex);
|
||||
//ÐкÅ
|
||||
@@ -82,6 +83,9 @@ public class ExcelUtils{
|
||||
}
|
||||
}
|
||||
ret.add(rowData);
|
||||
if (ret.size() % 10000 == 0) {
|
||||
println(ret.size())
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
|
||||
+2
-1
@@ -140,6 +140,7 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) {
|
||||
|
||||
if (StringUtils.isBlank(input)) {
|
||||
@@ -624,7 +625,7 @@ public class OpenApiClient {
|
||||
params.put("token", TOKEN)
|
||||
try {
|
||||
JSONObject result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/evaluate", params)
|
||||
if (result.getInt("code") == 200) {
|
||||
if (result && result.getInt("code") == 200) {
|
||||
if (result.containsKey("result")) {
|
||||
// if (result.get("result").get("credit") == 0) {
|
||||
// return null
|
||||
|
||||
+4
-4
@@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
|
||||
def city = "À³Îß"
|
||||
def destCity = "¼ÃÄÏ"
|
||||
def region = ""
|
||||
def destRegion = "À³Îß"
|
||||
def city = "쭤捺"
|
||||
def destCity = "쭤捺"
|
||||
def region = "섈적"
|
||||
def destRegion = "촙쏠"
|
||||
|
||||
|
||||
|
||||
|
||||
+1
@@ -26,6 +26,7 @@ class GaoDeApiUtils {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", GAO_DE_AK)
|
||||
params.put("location", location)
|
||||
params.put("extensions","all")
|
||||
String content
|
||||
try {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
|
||||
Reference in New Issue
Block a user