update
This commit is contained in:
+95
-7
@@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ServerAreaMap
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* Created by dxy on 2016/12/15.
|
||||
*/
|
||||
@@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch {
|
||||
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def DISTRICT_STAT_MAP = [:]
|
||||
|
||||
public void process(Object args) {
|
||||
def cities = ServerAreaMap.findAllCities()
|
||||
|
||||
|
||||
def cities = ["深圳"]//ServerAreaMap.findAllCities()
|
||||
for(String city : cities) {
|
||||
DISTRICT_STAT_MAP.clear()
|
||||
Map<String, List<String>> regions = RegionData.getAllRegions(city)
|
||||
regions.each {region, alias->
|
||||
processRegionRank(city, region)
|
||||
}
|
||||
processCityRank(city)
|
||||
// DISTRICT_STAT_MAP.clear()
|
||||
// Map<String, List<String>> regions = RegionData.getAllRegions(city)
|
||||
// regions.each {region, alias->
|
||||
// processRegionRank(city, region)
|
||||
// }
|
||||
// processCityRank(city)
|
||||
|
||||
processCityRentRank(city)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch {
|
||||
return ret
|
||||
}
|
||||
|
||||
def processCityRentRank(city){
|
||||
def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd")
|
||||
def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth)
|
||||
def prices = []
|
||||
distRentsMap.each{key, price->
|
||||
if(!prices.contains(price)){
|
||||
prices << price
|
||||
}
|
||||
}
|
||||
// prices = prices.unique()
|
||||
prices = prices.sort().reverse()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def keys = distRentsMap.keySet().asList()
|
||||
final CountDownLatch latch = new CountDownLatch(keys.size())
|
||||
|
||||
// for(def i=0;i<keys.size();i++){
|
||||
// final def key = keys[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
// totalCount.addAndGet(1)
|
||||
// upateDistrict(key, distRentsMap, prices)
|
||||
// println totalCount.get()+" "+ key
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
// latch.await()
|
||||
|
||||
def index = 0
|
||||
distRentsMap.each{key, price->
|
||||
index ++
|
||||
def arrs = key.split("##");
|
||||
double percentile = calculatePercentile(prices, price);
|
||||
|
||||
DBObject q = new BasicDBObject("city", arrs[0])
|
||||
q.append("region", arrs[1])
|
||||
q.append("district_name", arrs[2])
|
||||
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("rent_city_rank", (int)percentile);
|
||||
udata.put("m_date", new Date())
|
||||
DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
println index +" "+ arrs[2]
|
||||
}
|
||||
}
|
||||
|
||||
def upateDistrict(key, distRentsMap, prices){
|
||||
def arrs = key.split("##");
|
||||
def price = distRentsMap.get(key)
|
||||
|
||||
double percentile = calculatePercentile(prices, price);
|
||||
|
||||
DBObject q = new BasicDBObject("city", arrs[0])
|
||||
q.append("region", arrs[1])
|
||||
q.append("district_name", arrs[2])
|
||||
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("rent_city_rank", (int)percentile);
|
||||
udata.put("m_date", new Date())
|
||||
// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
}
|
||||
|
||||
public static int calculatePercentile(def values, def targetValue) {
|
||||
// 找出数字在List中的排名
|
||||
def rank = values.indexOf(targetValue) + 1
|
||||
// 计算百分位数
|
||||
def percentile = (int)((rank / values.size()) * 100)
|
||||
return percentile;
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictStatRankBatch(), args)
|
||||
|
||||
+4
-4
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
@@ -78,8 +78,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
|
||||
+45
-44
@@ -66,50 +66,51 @@ class BigdataDealBatch extends BaseBatch{
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
// saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
@@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(trades.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
@@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}
|
||||
}
|
||||
|
||||
+41
-3
@@ -28,9 +28,11 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
task.deleteThirdPartyData()
|
||||
// task.deleteThirdPartyData()//删除第三方
|
||||
// task.delDealByPe()
|
||||
// task.delDealData()
|
||||
|
||||
task.exportThirdParty()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -372,15 +374,29 @@ class DealPriceDataProcess {
|
||||
return map
|
||||
}
|
||||
|
||||
def exportThirdParty(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def results = []
|
||||
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty3(city, 202305)
|
||||
for(def map : rets){
|
||||
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
|
||||
}
|
||||
println city +" "+ rets.size()
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方")
|
||||
}
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202303
|
||||
def month = 202304
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202304)
|
||||
def rets = findPeGteThirdParty2(city, 202305)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
@@ -431,6 +447,28 @@ class DealPriceDataProcess {
|
||||
return rets
|
||||
}
|
||||
|
||||
//城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE
|
||||
|
||||
def findPeGteThirdParty3(cityName, yearmonth){
|
||||
def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15"
|
||||
def rets = []
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityid = r.urbn_id
|
||||
map.city = r.urbn_nm
|
||||
map.data_id = r.gujia_fsid
|
||||
map.region = r.gujia_rgon_nm
|
||||
map.comm_id = r.gujia_comm_id
|
||||
map.district_name = r.gujia_comm_nm
|
||||
map.type = r.gujia_bds_clc_tp
|
||||
map.clc_price = r.gujia_bds_avprc
|
||||
map.price = r.gujiabds_to_mkprc_mny
|
||||
map.ppe = r.gujia_mns_rto
|
||||
rets << map
|
||||
});
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def delDealData(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d ->
|
||||
|
||||
+3
-3
@@ -79,9 +79,9 @@ class ExportOracleAllDataTest {
|
||||
// exportCommCaseDetailGj()//
|
||||
|
||||
// // ok文件
|
||||
exportInOk()
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
|
||||
// exportInOk()
|
||||
// SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
|
||||
|
||||
}
|
||||
|
||||
|
||||
+12
-10
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
// task.updatePrice()
|
||||
// task.addFjwPrice()
|
||||
task.addFjwPrice()
|
||||
// task.addCcbPrice()//oracle
|
||||
|
||||
// task.addCcbPriceFromGuided()
|
||||
@@ -42,7 +42,7 @@ class FjwPricePkCcbPrice {
|
||||
// task.exportFjwPriceAndCcbPrice()
|
||||
// task.addNewDistPrice()
|
||||
|
||||
task.addCcbPrice2()//按id查
|
||||
// task.addCcbPrice2()//按id查
|
||||
// task.addCcbPrice3()//按city,region,name查
|
||||
// task.standRegion()
|
||||
// task.addGuidPrice()
|
||||
@@ -217,17 +217,19 @@ class FjwPricePkCcbPrice {
|
||||
}
|
||||
|
||||
def addFjwPrice(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d->
|
||||
if(!d['N']){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(dist){
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
d['K'] = dbo.avg_price
|
||||
}
|
||||
d['N'] = dist.region
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd"))
|
||||
// if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+4
-4
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -83,9 +83,9 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
//
|
||||
// //更新静态数据表
|
||||
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
|
||||
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
class FumingTest {
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
def task = new FumingTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run() {
|
||||
def path = "D:\\数据\\富民\\富民测试500-v7.xlsx"
|
||||
// match(path)
|
||||
// addPrice(path)
|
||||
// addThirdPrice(path)
|
||||
// addLastPrice(path)
|
||||
addFacLevel(path)
|
||||
}
|
||||
|
||||
|
||||
def addFacLevel(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['K'] && row['I']){
|
||||
// def obj = findCommunitySummary(row['C'])
|
||||
// if(obj){
|
||||
// row['K'] = obj.consumption_rating
|
||||
// }else{
|
||||
row['K'] = getNearByDistrictLevel(row['C'], row['I'])
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 一公里小区数量
|
||||
*/
|
||||
def getNearByDistrictLevel(def id, def price){
|
||||
BasicDBObject district = DbUtils.seaweed.findById(id);
|
||||
def level = null
|
||||
if(district){
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(district);
|
||||
println id +" 共:"+list.size()+" "+ price
|
||||
if(list!=null&&list.size()>0){
|
||||
for(def obj : list){
|
||||
def summary = findCommunitySummary(obj._id.toString())
|
||||
if(summary && summary.avg_price && summary.consumption_rating){
|
||||
def pre = java.lang.Math.abs((summary.avg_price - price)*100/price)
|
||||
println summary.name +" "+pre +" "+ summary.avg_price
|
||||
if(pre <= 10){
|
||||
level = summary.consumption_rating
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return level;
|
||||
}
|
||||
|
||||
|
||||
def addLastPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Z'] ){
|
||||
if(row['W']){
|
||||
row['Z'] = row['W']
|
||||
}else if(row['R']){
|
||||
row['Z'] = row['R']
|
||||
}else if(row['I']){
|
||||
row['Z'] = row['I']
|
||||
}
|
||||
|
||||
if(row['Z']){
|
||||
row['J'] = (int)(row['Z']*row['B'].toString().toDouble())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['N'] ){
|
||||
def city = row['C']
|
||||
def address = row['A']
|
||||
def region = row['D']
|
||||
def name = row['G']
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['N'] = jsonObject.get("id")
|
||||
row['O'] = jsonObject.get("city")
|
||||
row['P'] = jsonObject.get("district")
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['R'] = jsonObject.get("propertyType")
|
||||
row['S'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['C']
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
row['D'] = dist.city
|
||||
row['E'] = dist.region
|
||||
row['F'] = dist.name
|
||||
row['G'] = dist.estate_type2
|
||||
row['H'] = dist.address
|
||||
// row['Q'] = dist.name
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['I'] = priceObj.avg_price
|
||||
row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble())
|
||||
row['R'] = priceObj.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连']
|
||||
def allMap = [:]
|
||||
// for(def c : citys){
|
||||
// def cityMap = findThirdPartyPrice(c)
|
||||
// allMap << cityMap
|
||||
// }
|
||||
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
|
||||
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['L'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("beike_esf_price")){
|
||||
row['M'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("anjuke_esf_price")){
|
||||
row['N'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
def map = findAvgPrice(row['C'], cityCodeMap.get(row['D']))
|
||||
if(map){
|
||||
row['O'] = map.avg_price
|
||||
row['P'] = map.credit_value
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addCcbPrice2(){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
|
||||
if(d['C']){
|
||||
def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
|
||||
if(map){
|
||||
d['K'] = map.avg_price
|
||||
d['L'] = map.calc_type
|
||||
// d['AI'] = map.credit_value
|
||||
d['M'] = map.month
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def findCommunitySummary(id){
|
||||
def obj = null
|
||||
def sql = "select * from community_summary where id='${id}'"
|
||||
ElasticUtils.executeSqlQuery(sql, {
|
||||
obj = it
|
||||
})
|
||||
return obj
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -26,7 +26,7 @@ class MingShenPocTest {
|
||||
def run(){
|
||||
def path = "D:\\数据\\民生\\poc测试.xlsx"
|
||||
//9级标准化
|
||||
//structureV2(path)//针对海南poc地址数据此接口不合适使用
|
||||
structureV2(path)//针对海南poc地址数据此接口不合适使用
|
||||
//省市区拆分
|
||||
// ner(path)//城市,区域带市带区
|
||||
// match(path)
|
||||
@@ -34,7 +34,7 @@ class MingShenPocTest {
|
||||
// addLatLng(path)
|
||||
// addAroundPrice(path)
|
||||
// addAuthor(path)
|
||||
addHouse(path)
|
||||
// addHouse(path)
|
||||
}
|
||||
|
||||
def structureV2(path){
|
||||
|
||||
+30
-2
@@ -73,6 +73,8 @@ class CommonDatautils {
|
||||
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
|
||||
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -230,6 +232,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("name", "长滩美墅三期")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -781,7 +784,7 @@ class CommonDatautils {
|
||||
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
@@ -3418,11 +3421,27 @@ class CommonDatautils {
|
||||
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def sqmetreRentMap = [:]
|
||||
for(def d : dbcursor){
|
||||
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
|
||||
// sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
|
||||
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price)
|
||||
}
|
||||
return sqmetreRentMap;
|
||||
}
|
||||
|
||||
|
||||
static def findAllSqmetreRentPrices(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("cat","district")
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000)
|
||||
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def prices = []
|
||||
for(def d : dbcursor){
|
||||
prices << d.avg_price
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
static def findNewSqmetreRent(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
@@ -3490,5 +3509,14 @@ class CommonDatautils {
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
static def findDistrictAnalyStat(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q)
|
||||
return analyData
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+18
-8
@@ -82,20 +82,21 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
for(def dist : removeDists){
|
||||
def key = dist.city +"##"+dist.region+"##"+dist.name
|
||||
if(!distKeys.contains(key)){
|
||||
distKeys << key
|
||||
districts << dist
|
||||
}
|
||||
}
|
||||
// for(def dist : removeDists){
|
||||
// def key = dist.city +"##"+dist.region+"##"+dist.name
|
||||
// if(!distKeys.contains(key)){
|
||||
// distKeys << key
|
||||
// districts << dist
|
||||
// }
|
||||
// }
|
||||
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
|
||||
|
||||
def fzAddrMap = getFzAddress();
|
||||
|
||||
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
|
||||
saveOrUpdateFzDist(districts)
|
||||
// saveOrUpdateFzDist(districts)
|
||||
remarkRemoveDist(removeDists)
|
||||
def fzDistricts = getFzDistricts(city)
|
||||
|
||||
for(def rMap : fzDistricts){
|
||||
@@ -185,6 +186,15 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def remarkRemoveDist(removeDists){
|
||||
for(def rd : removeDists){
|
||||
def udate = new BasicDBObject()
|
||||
udate.put("status",1)
|
||||
udate.put("m_date", new Date())
|
||||
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
|
||||
}
|
||||
}
|
||||
|
||||
def getFzDistricts(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
+8
-6
@@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
if(weekendPriceObj.value_type!=1){
|
||||
if(dist.city == "¸£ÖÝ"){
|
||||
if(Math.abs(mom)*100>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
if(dist.region != "Âíβ"){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
}else{
|
||||
if(Math.abs(mom)*100>10){
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FzDataResultStat {
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
task.regionPriceTrendRun()
|
||||
// task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
@@ -11,26 +12,91 @@ import com.mongodb.DBCollection
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def datas = FileUtils.readLines("D:\\Êý¾Ý\\ÆäËü\\FJW_COMM_20221015.csv")
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def arrs = d.split("@")
|
||||
rets << ["commId":arrs[0], "avg_price":arrs[4]]
|
||||
}
|
||||
|
||||
for(def r : rets){
|
||||
if(r.avg_price){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("date", 202210)
|
||||
udata.put("avg", r.avg_price.toInteger())
|
||||
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
|
||||
}else{
|
||||
println r.commId +" ²»´æÔÚ"
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
|
||||
def city = RegionData.formatCity("余姚")
|
||||
d['D'] = city
|
||||
if(city){
|
||||
d['E'] = RegionData.formatRegion(city, d['C'])
|
||||
}
|
||||
|
||||
// if(!d['D']){
|
||||
// def citys = CommonDatautils.getCityByRegion(d['A'])
|
||||
// if(citys && citys.size()==1){
|
||||
// d['F'] = citys.get(0)
|
||||
// }
|
||||
// }
|
||||
})
|
||||
|
||||
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
|
||||
//def m = [:]
|
||||
//for(def d : datas){
|
||||
// m.put(d['B'], d)
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d->
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// if(dist){
|
||||
// def ret = m.get(dist._id.toString())
|
||||
// if(ret){
|
||||
// d['E'] = ret['J']
|
||||
// d['F'] = ret['N']
|
||||
// }
|
||||
|
||||
// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS)
|
||||
// if(dbo){
|
||||
// if(dbo.beike_esf_price){
|
||||
// d['G'] = dbo.beike_esf_price
|
||||
// }else if(dbo.beike_esf_price){
|
||||
// d['G'] = dbo.beike_esf_price
|
||||
// }
|
||||
//
|
||||
// if(dbo.maitian_esf_price){
|
||||
// d['H'] = dbo.maitian_esf_price
|
||||
// }
|
||||
// }
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd"))
|
||||
// if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
// d['L'] = dbo.value_type
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", 202304)
|
||||
return AVERAGE_STATISTICS.findOne(q)
|
||||
}
|
||||
//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
|
||||
//def rets = []
|
||||
//for(def d : datas){
|
||||
// def arrs = d.split("@")
|
||||
// rets << ["commId":arrs[0], "avg_price":arrs[4]]
|
||||
//}
|
||||
//
|
||||
//for(def r : rets){
|
||||
// if(r.avg_price){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("date", 202210)
|
||||
// udata.put("avg", r.avg_price.toInteger())
|
||||
// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
|
||||
// }else{
|
||||
// println r.commId +" 不存在"
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
+5
-2
@@ -29,8 +29,11 @@ class AddAvgpriceBatch {
|
||||
}
|
||||
|
||||
def test(){
|
||||
//发重算均价命令
|
||||
addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd'))
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"])
|
||||
for(def dist : districts){
|
||||
//发重算均价命令
|
||||
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
|
||||
|
||||
+13
-2
@@ -96,17 +96,28 @@ class CityStatExport {
|
||||
}
|
||||
|
||||
def regionStat(){
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州']
|
||||
def ret = []
|
||||
ret << ["城市","区域","小区总量"]
|
||||
for(def city : citys){
|
||||
|
||||
def regions = CommonDatautils.getRegionListByCity(city)
|
||||
for(def r : regions){
|
||||
def row = []
|
||||
row << city
|
||||
row << r
|
||||
def districts = CommonDatautils.findDistricts(city, r, null)
|
||||
println city +" "+ r +" "+ districts.size()
|
||||
|
||||
row << CommonDatautils.queryCityRegionPrice(city, r, null)
|
||||
def obj = CommonDatautils.findRegionPriceRate(city, r)
|
||||
row << obj.price_coverage*100
|
||||
row << districts.size()
|
||||
|
||||
ret << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
|
||||
ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县")
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -272,6 +272,45 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processName(name){
|
||||
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
|
||||
if (name.contains('(') && name.contains(')')){
|
||||
def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.contains('(') && name.contains(')')){
|
||||
def rep_str = name.subsring(name.index('('),name.rfind(')')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('【')!=-1 && name.find('】')!=-1){
|
||||
def rep_str = name[name.index('【'):name.rfind('】')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('【')!=-1 && name.find('】')!=-1){
|
||||
def rep_str = name[name.index('【'):name.rfind('】')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('(')!=-1 && name.find(')')!=-1){
|
||||
def rep_str = name[name.index('('):name.rfind(')')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('「')!=-1 && name.find('」')!=-1){
|
||||
def rep_str = name[name.index('「'):name.rfind('」')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
static def processCarSite(car_site_count){
|
||||
def v = getValue(car_site_count.toString());
|
||||
if(v){
|
||||
@@ -579,6 +618,12 @@ class FiledStrand {
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}else if(estateType2=="写字楼"){
|
||||
if(estateCharege>=0.35 && estateCharege<=60){
|
||||
return estateCharege;
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
println "estateCharege ת²»ÁË "+estateCharege
|
||||
|
||||
@@ -567,7 +567,7 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
static def financialRating(city, address){
|
||||
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3"
|
||||
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
@@ -580,5 +580,79 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
|
||||
//源地址的完整性进行验证
|
||||
static def addrVerify(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// 省、市、区、街道、社区居委、POI补全
|
||||
static def complete(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
//五级(省/市/区/街道/社区居委)解析、补全
|
||||
static def division(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//识别地址中省、市、区、道路、POI同时进行地址验证
|
||||
static def structure(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 招商银行测试
|
||||
* @param city
|
||||
* @param address
|
||||
* @return
|
||||
*/
|
||||
static def cmbTest(city, address){
|
||||
def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
|
||||
if(d['A'] && !d['E']){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
|
||||
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
|
||||
println d['A'] +" "+d['C']
|
||||
// def total = findDatas(d['A'], d['B'], d['C'])
|
||||
// d['D'] = total
|
||||
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
|
||||
d['E'] = c
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
def getAmapLocation(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name.contains(name) && d.type.contains("住宅")){
|
||||
total ++
|
||||
}
|
||||
})
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
def findDatas(def city, def region, def name){
|
||||
def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'"
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
total = d.count
|
||||
})
|
||||
return total;
|
||||
}
|
||||
|
||||
//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
|
||||
// def region = RegionData.formatCity(d['A'])
|
||||
// if(!region && d['A']){
|
||||
// println d['A']
|
||||
// }
|
||||
//})
|
||||
@@ -76,24 +76,24 @@ MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S
|
||||
|
||||
|
||||
|
||||
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
hashSet << result
|
||||
}
|
||||
|
||||
|
||||
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
//def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
//def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
|
||||
//Calendar calendar = Calendar.getInstance()
|
||||
//calendar.setTime(startDate)
|
||||
//def count = 0
|
||||
//def hashSet = []
|
||||
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
//
|
||||
//OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
// def result = JSONObject.fromObject(doc.res_map.result)
|
||||
// def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
// data << item
|
||||
// hashSet << result
|
||||
//}
|
||||
//
|
||||
//
|
||||
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
//ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
|
||||
@@ -115,4 +115,27 @@ ExcelUtils.write("D:\\
|
||||
//
|
||||
//
|
||||
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
|
||||
|
||||
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230524", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]]
|
||||
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode,
|
||||
doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber]
|
||||
data << item
|
||||
hashSet << result
|
||||
}
|
||||
|
||||
|
||||
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
@@ -0,0 +1,207 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 按区域统计覆盖率
|
||||
*/
|
||||
|
||||
class XiamenStat {
|
||||
|
||||
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def DISTRICT_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
|
||||
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
def task = new XiamenStat();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d->
|
||||
if(d['D'] && d['D']=="全市"){
|
||||
def dbo = CommonDatautils.findCityInfo(d['C'].replace("市",""))
|
||||
if(dbo){
|
||||
def detail = dbo.district_detail
|
||||
if(detail){
|
||||
d['E'] = dbo.existing_district_count//数据量
|
||||
d['F'] = detail.township*100// 乡镇街道
|
||||
d['G'] = detail.address*100// 地址
|
||||
d['H'] = detail.addrAlias*100// 地址别名
|
||||
d['I'] = detail.name*100// 名称
|
||||
d['J'] = detail.alias*100// 别名
|
||||
d['K'] = detail.alias*100// 映射别名
|
||||
d['L'] = detail.lat2*100// 经纬度(百度)
|
||||
d['M'] = detail.lat*100// 经纬度(高德)
|
||||
d['N'] = detail.estateType2*100// 物业类型
|
||||
d['O'] = detail.buildType*100// 建筑类型
|
||||
d['P'] = detail.streetCommunity*100// 街道社区
|
||||
d['Q'] = detail.committee*100// 居委会
|
||||
d['R'] = detail.businessQuarter*100// 商圈
|
||||
d['S'] = detail.schools*100// 对口小学
|
||||
d['T'] = detail.kindergartens*100// 对口幼儿园
|
||||
d['U'] = detail.loudongCount*100// 楼栋总数
|
||||
d['V'] = detail.householdCount*100// 户数
|
||||
d['W'] = detail.estateCharge*100// 物业费
|
||||
d['X'] = detail.carSiteCount*100// 停车位
|
||||
d['Y'] = ""// 电梯
|
||||
d['Z'] = detail.propertyLife*100// 产权年限
|
||||
d['AA'] = detail.houseProperty*100// 产权性质
|
||||
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
|
||||
d['AC'] = detail.landProperty*100// 小区性质
|
||||
d['AD'] = detail.finishBuildingDate*100// 竣工时间
|
||||
d['AE'] = detail.openDate*100// 交房时间
|
||||
d['AF'] = detail.allArea*100// 占地面积
|
||||
d['AG'] = detail.buildArea*100// 建筑面积
|
||||
d['AH'] = detail.cubagePercent*100// 容积率
|
||||
d['AI'] = detail.virescencePercent*100// 绿化率
|
||||
d['AJ'] = detail.mainType*100// 主力户型
|
||||
d['AK'] = detail.mainTypeArea*100// 主力户型面积
|
||||
d['AL'] = detail.developer*100// 开发商
|
||||
d['AM'] = detail.estateCompany*100// 物业公司
|
||||
d['AN'] = dbo.price_coverage*100// 小区当前均价
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def fields = DISTRICT_FIELDS_MAP.keySet().asList()
|
||||
def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields)
|
||||
def detail = districtDetail(regionDistricts.size(), regionDistricts);
|
||||
if(detail){
|
||||
d['E'] = regionDistricts.size()//数据量
|
||||
d['F'] = detail.township*100// 乡镇街道
|
||||
d['G'] = detail.address*100// 地址
|
||||
d['H'] = detail.addrAlias*100// 地址别名
|
||||
d['I'] = detail.name*100// 名称
|
||||
d['J'] = detail.alias*100// 别名
|
||||
d['K'] = detail.alias*100// 映射别名
|
||||
d['L'] = detail.lat2*100// 经纬度(百度)
|
||||
d['M'] = detail.lat*100// 经纬度(高德)
|
||||
d['N'] = detail.estateType2*100// 物业类型
|
||||
d['O'] = detail.buildType*100// 建筑类型
|
||||
d['P'] = detail.streetCommunity*100// 街道社区
|
||||
d['Q'] = detail.committee*100// 居委会
|
||||
d['R'] = detail.businessQuarter*100// 商圈
|
||||
d['S'] = detail.schools*100// 对口小学
|
||||
d['T'] = detail.kindergartens*100// 对口幼儿园
|
||||
d['U'] = detail.loudongCount*100// 楼栋总数
|
||||
d['V'] = detail.householdCount*100// 户数
|
||||
d['W'] = detail.estateCharge*100// 物业费
|
||||
d['X'] = detail.carSiteCount*100// 停车位
|
||||
d['Y'] = ""// 电梯
|
||||
d['Z'] = detail.propertyLife*100// 产权年限
|
||||
d['AA'] = detail.houseProperty*100// 产权性质
|
||||
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
|
||||
d['AC'] = detail.landProperty*100 // 小区性质
|
||||
d['AD'] = detail.finishBuildingDate*100// 竣工时间
|
||||
d['AE'] = detail.openDate*100// 交房时间
|
||||
d['AF'] = detail.allArea*100// 占地面积
|
||||
d['AG'] = detail.buildArea*100// 建筑面积
|
||||
d['AH'] = detail.cubagePercent*100// 容积率
|
||||
d['AI'] = detail.virescencePercent*100// 绿化率
|
||||
d['AJ'] = detail.mainType*100// 主力户型
|
||||
d['AK'] = detail.mainTypeArea*100// 主力户型面积
|
||||
d['AL'] = detail.developer*100// 开发商
|
||||
d['AM'] = detail.estateCompany*100// 物业公司
|
||||
// d['AN'] = ""// 小区当前均价
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def districtDetail(def disCount, def regionDistricts) {
|
||||
|
||||
def countMap = [:]
|
||||
for(def dbo : regionDistricts) {
|
||||
DISTRICT_FIELDS_MAP.each() {key,skey->
|
||||
if (isValid(dbo.get(key))) {
|
||||
if(countMap.containsKey(key)) {
|
||||
countMap.put(key, countMap.get(key) + 1)
|
||||
}else {
|
||||
countMap.put(key, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def districtDetail = [:];
|
||||
DISTRICT_FIELDS_MAP.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
districtDetail.put(skey, roundDouble(countMap.get(key), disCount))
|
||||
}else {
|
||||
districtDetail.put(skey, 0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(countMap.containsKey("lat") && countMap.containsKey("lng")) {
|
||||
districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount));
|
||||
}else {
|
||||
districtDetail.put('latLng', 0);
|
||||
}
|
||||
|
||||
if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) {
|
||||
districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount));
|
||||
}else {
|
||||
districtDetail.put('latLng2', 0);
|
||||
}
|
||||
return districtDetail;
|
||||
}
|
||||
|
||||
|
||||
def CONTAINS_KWS = ["...", "暂无", "待定", "未知"]
|
||||
def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."]
|
||||
def isValid(def value) {
|
||||
if (!value) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (value instanceof String) {
|
||||
value = value.trim()
|
||||
for(String ck : CONTAINS_KWS) {
|
||||
if(value.contains(ck)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for(String ek : EQUALS_KWS) {
|
||||
if(ek.equals(value)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Number) {
|
||||
if (value <= 0) {
|
||||
return false;
|
||||
}
|
||||
} else if (value instanceof Collection || value instanceof Map) {
|
||||
if (value.size() == 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
double roundDouble(int a, int base) {
|
||||
if(a == 0 || base == 0) {
|
||||
return 0
|
||||
}
|
||||
return ((double)a / (double)base * 10000.0).toInteger() / 10000.0
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = ["ÕÄÖÝ"]//CommonDatautils.findAllCity().reverse()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
|
||||
def analysRets = []
|
||||
|
||||
|
||||
+1
-1
@@ -95,7 +95,7 @@ class FangjiaClientUtils {
|
||||
try {
|
||||
if(address){
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 写字楼字段清洗入库(自动入库)
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static main(args) {
|
||||
runBatch(new OfficeFieldCleanBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
for(def c : citys){
|
||||
def offices = findAllOffice(c)
|
||||
strandField(offices)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def strandField(offices){
|
||||
def fields = ["property_fee","build_area","","","","","","","","","",""]
|
||||
for(def office : offices){
|
||||
fields.each{
|
||||
def newVal = null
|
||||
if(it == "property_fee"){
|
||||
newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼")
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
|
||||
if(it =="all_area" || it =="build_area"){
|
||||
newVal = FiledStrand.processArea(office.get(it))
|
||||
if(newVal && newVal > 5000){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
|
||||
if(it =="name"){
|
||||
newVal = FiledStrand.processName(office.get(it))
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findAllOffice(city){
|
||||
def sql = "select * from office where city='${city}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+3
@@ -144,5 +144,8 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
|
||||
|
||||
public static void main(def agrs){
|
||||
doMatch("上海", "静安", "金家巷西弄", null)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+12
-5
@@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&
|
||||
stat()
|
||||
|
||||
def stat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1)
|
||||
def map=[:]
|
||||
for(def d : datas){
|
||||
def count = map.get(d['C'])
|
||||
@@ -69,8 +69,8 @@ def findLogs(){
|
||||
|
||||
def findFangjiaLogs(){
|
||||
|
||||
def startDate = DateUtil.parseDate("20230329", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230429", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230531", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def list = []
|
||||
@@ -84,14 +84,21 @@ def findFangjiaLogs(){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
|
||||
// def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
|
||||
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
|
||||
|
||||
def q = QueryBuilders.fuzzyQuery("request", "/enquire")
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
|
||||
println houses.size()
|
||||
|
||||
if (houses) {
|
||||
for (def d : houses) {
|
||||
def reqStr = d.request.toString()
|
||||
if(!reqStr.contains("/enquire")){
|
||||
continue
|
||||
}
|
||||
def mid = null
|
||||
if (reqStr.contains("mid=")) {
|
||||
|
||||
@@ -107,5 +114,5 @@ def findFangjiaLogs(){
|
||||
startDate = nextDate
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志")
|
||||
ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志")
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class HouseAnaly {
|
||||
|
||||
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new HouseAnaly()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allRets = []
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"])
|
||||
def rets = processing(districts)
|
||||
println "[${city}] 写文件 "+rets.size()
|
||||
allRets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析")
|
||||
}
|
||||
|
||||
def processing(dists){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"]
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(dist.alias && dist.alias.size()>=1){
|
||||
def ret = analy(dist)
|
||||
if(ret){
|
||||
rets.addAll(ret)
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def analy(dist){
|
||||
def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是");
|
||||
def alias = dist.alias
|
||||
for(def aliasName : alias){
|
||||
if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){
|
||||
def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否");
|
||||
if(aliaRet){
|
||||
masterRet.addAll(aliaRet)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(masterRet.size()>=2){
|
||||
def masterAvgPrice = 0
|
||||
def masterMidPrice = 0
|
||||
def back = false
|
||||
for(def row : masterRet){
|
||||
if(row.get(4)=="是"){
|
||||
masterAvgPrice = row.get(8)
|
||||
masterMidPrice = row.get(9)
|
||||
}else{
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
if(masterAvgPrice>0){
|
||||
pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice)
|
||||
}
|
||||
if(masterMidPrice>0){
|
||||
pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice)
|
||||
}
|
||||
if(pre1>=20 || pre2>=20){
|
||||
back = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if(back){
|
||||
return masterRet
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量
|
||||
def runAnalying(id, city, region, name, flag){
|
||||
def houses = findHouse(city, region, name);
|
||||
def rets = []
|
||||
if(houses && houses.size()>=5){
|
||||
def prices = []
|
||||
for(def house : houses){
|
||||
prices << house.avg_price
|
||||
}
|
||||
prices = prices.sort()
|
||||
def average = (int)(prices.sum() / prices.size())
|
||||
def midPrice = getMidPrice(prices)
|
||||
def price10 = null
|
||||
def price90 = null
|
||||
if(houses.size()>=10){
|
||||
def postion = Math.round(houses.size()/10)
|
||||
price10 = prices.get((int)(postion-1))
|
||||
price90 = prices.get((int)(houses.size()-postion-1))
|
||||
}
|
||||
def countMap = classType(houses)
|
||||
|
||||
// def pre = java.lang.Math.abs((midPrice-average)*100/average)
|
||||
// if(pre>=20){
|
||||
rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")]
|
||||
// }
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def classType(houses){
|
||||
def district = 0
|
||||
def villa = 0
|
||||
def business = 0
|
||||
for(def h : houses){
|
||||
if(h.estate_type){
|
||||
if(h.estate_type == "别墅"){
|
||||
villa = villa + 1
|
||||
continue
|
||||
}else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){
|
||||
business = business + 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
def valla = isVilla(h.area, h.name, h.height, h.room)
|
||||
if(valla){
|
||||
villa = villa + 1
|
||||
continue
|
||||
}
|
||||
district = district + 1
|
||||
}
|
||||
return ["district":district, "villa":villa, "business":business]
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public boolean isVilla(area, name, height, room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
def getMidPrice(def prices){
|
||||
def mid = prices.size()/2
|
||||
if(mid > (int)mid){
|
||||
return prices.get((int)mid)
|
||||
}else{
|
||||
def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1))
|
||||
return (int)(total/2)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def findHouse(def city, def region, def name){
|
||||
def houses = []
|
||||
try{
|
||||
name = escapeExprSpecialWord(name)
|
||||
String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'"
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type]
|
||||
houses << r
|
||||
// println r
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return houses
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
|
||||
}
|
||||
+13
-3
@@ -21,13 +21,23 @@ class GaoDeMapSearchTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new GaoDeMapSearchTest();
|
||||
def path = "D:\\数据\\光大\\光大测试-V2.xlsx"
|
||||
def path = "D:\\数据\\富民\\富民测试500.xlsx"
|
||||
|
||||
// task.structureV2(path)
|
||||
// task.match(path)
|
||||
// task.mapSearch(path)
|
||||
task.baiduMapSearch(path)
|
||||
task.checkTrue(path)
|
||||
// task.baiduMapSearch(path)
|
||||
// task.checkTrue(path)
|
||||
task.addFinancialLevel(path)
|
||||
}
|
||||
|
||||
def addFinancialLevel(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A'])
|
||||
if(results){
|
||||
row['V'] = results.consumptionRating
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def checkTrue(path){
|
||||
|
||||
+67
-27
@@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome {
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
|
||||
//广州,佛山 重庆
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
@@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome {
|
||||
println city +" 已运行"
|
||||
continue
|
||||
}
|
||||
def newCity = RegionData.formatCity(city)
|
||||
if(!newCity){
|
||||
if(city=="昆山"){
|
||||
newCity = "苏州"
|
||||
}
|
||||
println city +" 无法格式化"
|
||||
}
|
||||
def url = m[1]
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
@@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome {
|
||||
def href = url+ r.attr("href")
|
||||
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, city, href, regionText)
|
||||
if(city == "昆山"){
|
||||
regionText = "昆山"
|
||||
}
|
||||
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
@@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, region)
|
||||
def dbo = findTrade(durl , "贝壳找房")
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, region){
|
||||
def parse(city,e, region, driver){
|
||||
try {
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
if(title.contains("车位")){
|
||||
detail.type = "车位"
|
||||
}
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
|
||||
@@ -266,28 +289,42 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
|
||||
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡",""))
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
detail.url = detailUrl
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
detail.url = detailUrl
|
||||
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
|
||||
detail.floor_info = floorStr
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorInfo && floorInfo.isNumber()) {
|
||||
detail.height = floorInfo.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -392,16 +429,16 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def findTrade(def url, source){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city", detail.city)
|
||||
// q.put("region", detail.region)
|
||||
// q.put("district_name", detail.district_name)
|
||||
// q.put("area", detail.area)
|
||||
// q.put("total_price", detail.total_price)
|
||||
q.put("url", detail.url)
|
||||
q.put("url", url)
|
||||
// q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
q.put("source", source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
@@ -446,12 +483,15 @@ public class BeiKeCrawlByChrome {
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
Thread.sleep(20000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
Thread.sleep(15000)
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
class DataFormatUtils {
|
||||
|
||||
static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000]
|
||||
|
||||
static def parseLift(def str){
|
||||
if(str && str.contains("梯")){
|
||||
def lift = str.toString().substring(0, str.toString().indexOf("梯"))
|
||||
if(lift){
|
||||
def val = chineseIntegerPart2Digit(lift)
|
||||
if(val){
|
||||
return (int)val
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def parseHousehold(def str){
|
||||
if(str && str.contains("梯") && str.contains("户")){
|
||||
def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户"))
|
||||
def val = chineseIntegerPart2Digit(household)
|
||||
if(val){
|
||||
return (int)val
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def chineseIntegerPart2Digit(String chineseDigit)
|
||||
{
|
||||
Double finalDigit = 0.0;
|
||||
Integer r = 1;
|
||||
|
||||
for(int i = chineseDigit.length()-1; i>=0;i--)
|
||||
{
|
||||
Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i)));
|
||||
|
||||
if( val >= 10 && i == 0 )
|
||||
{
|
||||
if (val > r)
|
||||
{
|
||||
r = val;
|
||||
finalDigit = finalDigit +val;
|
||||
}
|
||||
else
|
||||
{
|
||||
r = r * val;
|
||||
}
|
||||
}
|
||||
else if(val >= 10)
|
||||
{
|
||||
if (val > r)
|
||||
{
|
||||
r = val;
|
||||
}
|
||||
else
|
||||
{
|
||||
r = r * val;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
finalDigit = finalDigit + r*val;
|
||||
}
|
||||
}
|
||||
return finalDigit;
|
||||
}
|
||||
|
||||
static String getValue(String s) {
|
||||
if (s && s.contains("暂无")) {
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
+103
-64
@@ -196,9 +196,22 @@ public class LianjiaCrawlByChrome {
|
||||
println city +" 已运行"
|
||||
continue
|
||||
}
|
||||
def newCity = RegionData.formatCity(city)
|
||||
if(!newCity){
|
||||
if(city=="昆山"){
|
||||
newCity = "苏州"
|
||||
}
|
||||
println city +" 无法格式化"
|
||||
}
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
driver.get(newUrl)
|
||||
try{
|
||||
driver.get(newUrl)
|
||||
}catch(Exception e){
|
||||
println newUrl +" 链接有问题!"
|
||||
continue
|
||||
}
|
||||
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
@@ -208,9 +221,11 @@ public class LianjiaCrawlByChrome {
|
||||
for(def r : regions){
|
||||
def regionText = r.text()
|
||||
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
|
||||
|
||||
if(city == "昆山"){
|
||||
regionText = "昆山"
|
||||
}
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, city, href, regionText)
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
@@ -284,7 +299,10 @@ public class LianjiaCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, regionText)
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -302,85 +320,106 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, regionText){
|
||||
def parse(city,e, regionText, driver){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
if(regionText){
|
||||
detail.region = regionText
|
||||
}else{
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
}
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
def room = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
if(room && room.isNumber()){
|
||||
detail.room = room.toInteger()
|
||||
}
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
if(title.contains("车位")){
|
||||
detail.type = "车位"
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
if(hall && hall.isNumber()){
|
||||
detail.hall = hall.toInteger()
|
||||
}
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
if(regionText){
|
||||
detail.region = regionText
|
||||
}else{
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
}
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
}
|
||||
|
||||
if("车位"!=roomStr){
|
||||
if(arrs && arrs.size()>=3){
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
def room = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
if(room && room.isNumber()){
|
||||
detail.room = room.toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
if(hall && hall.isNumber()){
|
||||
detail.hall = hall.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
}
|
||||
if("车位"!=roomStr){
|
||||
if(arrs && arrs.size()>=3){
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = detailUrl
|
||||
|
||||
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层",""))
|
||||
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
|
||||
detail.floor_info = floorStr
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr2 && floorStr2.isNumber()) {
|
||||
detail.height = floorStr2.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -484,16 +523,16 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def findTrade(def url, source){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city", detail.city)
|
||||
// q.put("region", detail.region)
|
||||
// q.put("district_name", detail.district_name)
|
||||
// q.put("area", detail.area)
|
||||
// q.put("total_price", detail.total_price)
|
||||
q.put("url", url)
|
||||
// q.put("trade_date", detail.trade_date)
|
||||
q.put("url", detail.url)
|
||||
q.put("source", detail.source)
|
||||
q.put("source", source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
@@ -535,7 +574,7 @@ public class LianjiaCrawlByChrome {
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
Thread.sleep(15000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
}
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.fangjia.script.gj.region
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 格式化其它网站区域
|
||||
*/
|
||||
|
||||
|
||||
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
|
||||
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl)
|
||||
|
||||
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1)
|
||||
for(def d : datas){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("recognizeCity", d['F'])
|
||||
q.put("recognizeRegion", d['A'])
|
||||
def count = SERVER_AREA_COLL.count(q)
|
||||
if(!count){
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("city", d['A'])
|
||||
obj.put("recognizeRegion", d['A'])
|
||||
obj.put("recognizeCity", d['F'])
|
||||
obj.put("createdDate", new Date())
|
||||
SERVER_AREA_COLL.save(obj)
|
||||
}else{
|
||||
println d['A']+" "+ d['F'] +" 已存在"
|
||||
}
|
||||
}
|
||||
@@ -237,7 +237,7 @@ public class DbUtils{
|
||||
def lng = district.lng / 10000000000.0
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
|
||||
sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0));
|
||||
query.append('$sd', sdq)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
|
||||
Reference in New Issue
Block a user