Merge branch 'script_gj' into 'master'

update

See merge request !656
This commit is contained in:
2023-06-09 11:11:19 +08:00
34 changed files with 1669 additions and 237 deletions
@@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ServerAreaMap
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* Created by dxy on 2016/12/15.
*/
@@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch {
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def DISTRICT_STAT_MAP = [:]
public void process(Object args) {
def cities = ServerAreaMap.findAllCities()
def cities = ["深圳"]//ServerAreaMap.findAllCities()
for(String city : cities) {
DISTRICT_STAT_MAP.clear()
Map<String, List<String>> regions = RegionData.getAllRegions(city)
regions.each {region, alias->
processRegionRank(city, region)
}
processCityRank(city)
// DISTRICT_STAT_MAP.clear()
// Map<String, List<String>> regions = RegionData.getAllRegions(city)
// regions.each {region, alias->
// processRegionRank(city, region)
// }
// processCityRank(city)
processCityRentRank(city)
}
}
@@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch {
return ret
}
def processCityRentRank(city){
def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd")
def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth)
def prices = []
distRentsMap.each{key, price->
if(!prices.contains(price)){
prices << price
}
}
// prices = prices.unique()
prices = prices.sort().reverse()
def totalCount = new AtomicLong()
def keys = distRentsMap.keySet().asList()
final CountDownLatch latch = new CountDownLatch(keys.size())
// for(def i=0;i<keys.size();i++){
// final def key = keys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
// totalCount.addAndGet(1)
// upateDistrict(key, distRentsMap, prices)
// println totalCount.get()+" "+ key
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// })
// }
// latch.await()
def index = 0
distRentsMap.each{key, price->
index ++
def arrs = key.split("##");
double percentile = calculatePercentile(prices, price);
DBObject q = new BasicDBObject("city", arrs[0])
q.append("region", arrs[1])
q.append("district_name", arrs[2])
DBObject udata = new BasicDBObject()
udata.put("rent_city_rank", (int)percentile);
udata.put("m_date", new Date())
DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
println index +" "+ arrs[2]
}
}
def upateDistrict(key, distRentsMap, prices){
def arrs = key.split("##");
def price = distRentsMap.get(key)
double percentile = calculatePercentile(prices, price);
DBObject q = new BasicDBObject("city", arrs[0])
q.append("region", arrs[1])
q.append("district_name", arrs[2])
DBObject udata = new BasicDBObject()
udata.put("rent_city_rank", (int)percentile);
udata.put("m_date", new Date())
// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
public static int calculatePercentile(def values, def targetValue) {
// 找出数字在List中的排名
def rank = values.indexOf(targetValue) + 1
// 计算百分位数
def percentile = (int)((rank / values.size()) * 100)
return percentile;
}
static main(args) {
runBatch(new DistrictStatRankBatch(), args)
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
@@ -78,8 +78,8 @@ class BigdataCommunityBatch extends BaseBatch{
println "txt 生成->导Oracle"
delPgData()
saveToPgSql(rows)
// delPgData()
// saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
@@ -66,50 +66,51 @@ class BigdataDealBatch extends BaseBatch{
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
}
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
saveToPgSql(rows)
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
// saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
final CountDownLatch latch = new CountDownLatch(trades.size())
def final AtomicInteger count_ = new AtomicInteger(0)
@@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
}
}
if(rMap){
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}
}
@@ -28,9 +28,11 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
task.deleteThirdPartyData()
// task.deleteThirdPartyData()//删除第三方
// task.delDealByPe()
// task.delDealData()
task.exportThirdParty()
}
def delDealByPe(){
@@ -372,15 +374,29 @@ class DealPriceDataProcess {
return map
}
def exportThirdParty(){
def citys = CcbCommonUtils.citys_all
def results = []
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
for(def city : citys) {
def rets = findPeGteThirdParty3(city, 202305)
for(def map : rets){
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
}
println city +" "+ rets.size()
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方")
}
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202303
def month = 202304
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202304)
def rets = findPeGteThirdParty2(city, 202305)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -431,6 +447,28 @@ class DealPriceDataProcess {
return rets
}
//城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE
def findPeGteThirdParty3(cityName, yearmonth){
def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15"
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
map.cityid = r.urbn_id
map.city = r.urbn_nm
map.data_id = r.gujia_fsid
map.region = r.gujia_rgon_nm
map.comm_id = r.gujia_comm_id
map.district_name = r.gujia_comm_nm
map.type = r.gujia_bds_clc_tp
map.clc_price = r.gujia_bds_avprc
map.price = r.gujiabds_to_mkprc_mny
map.ppe = r.gujia_mns_rto
rets << map
});
return rets
}
def delDealData(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d ->
@@ -79,9 +79,9 @@ class ExportOracleAllDataTest {
// exportCommCaseDetailGj()//
// // ok文件
exportInOk()
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
// exportInOk()
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
// task.addFjwPrice()
task.addFjwPrice()
// task.addCcbPrice()//oracle
// task.addCcbPriceFromGuided()
@@ -42,7 +42,7 @@ class FjwPricePkCcbPrice {
// task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice()
task.addCcbPrice2()//按id查
// task.addCcbPrice2()//按id查
// task.addCcbPrice3()//按city,region,name查
// task.standRegion()
// task.addGuidPrice()
@@ -217,17 +217,19 @@ class FjwPricePkCcbPrice {
}
def addFjwPrice(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['C'])
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d->
if(!d['N']){
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd"))
if(dbo){
d['K'] = dbo.avg_price
}
d['N'] = dist.region
// def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd"))
// if(dbo){
// d['K'] = dbo.avg_price
// }
}
}
})
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -83,9 +83,9 @@ class GuiDingPriceBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
saveToPgSql(rows)
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
//
// //更新静态数据表
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
citys = CcbCommonUtils.getRunningCitys(args)
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
@@ -0,0 +1,261 @@
package com.fangjia.script.chongqing
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
class FumingTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new FumingTest();
task.run();
}
def run() {
def path = "D:\\数据\\富民\\富民测试500-v7.xlsx"
// match(path)
// addPrice(path)
// addThirdPrice(path)
// addLastPrice(path)
addFacLevel(path)
}
def addFacLevel(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['K'] && row['I']){
// def obj = findCommunitySummary(row['C'])
// if(obj){
// row['K'] = obj.consumption_rating
// }else{
row['K'] = getNearByDistrictLevel(row['C'], row['I'])
// }
}
})
}
/**
* 一公里小区数量
*/
def getNearByDistrictLevel(def id, def price){
BasicDBObject district = DbUtils.seaweed.findById(id);
def level = null
if(district){
List<DBObject> list = DbUtils.findAroundDistrict(district);
println id +" 共:"+list.size()+" "+ price
if(list!=null&&list.size()>0){
for(def obj : list){
def summary = findCommunitySummary(obj._id.toString())
if(summary && summary.avg_price && summary.consumption_rating){
def pre = java.lang.Math.abs((summary.avg_price - price)*100/price)
println summary.name +" "+pre +" "+ summary.avg_price
if(pre <= 10){
level = summary.consumption_rating
break;
}
}
}
}
}
return level;
}
def addLastPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Z'] ){
if(row['W']){
row['Z'] = row['W']
}else if(row['R']){
row['Z'] = row['R']
}else if(row['I']){
row['Z'] = row['I']
}
if(row['Z']){
row['J'] = (int)(row['Z']*row['B'].toString().toDouble())
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['N'] ){
def city = row['C']
def address = row['A']
def region = row['D']
def name = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['N'] = jsonObject.get("id")
row['O'] = jsonObject.get("city")
row['P'] = jsonObject.get("district")
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("propertyType")
row['S'] = jsonObject.get("address")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['C']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['D'] = dist.city
row['E'] = dist.region
row['F'] = dist.name
row['G'] = dist.estate_type2
row['H'] = dist.address
// row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['I'] = priceObj.avg_price
row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble())
row['R'] = priceObj.avg_price_2b
}
}
}
})
}
def addThirdPrice(path){
// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连']
def allMap = [:]
// for(def c : citys){
// def cityMap = findThirdPartyPrice(c)
// allMap << cityMap
// }
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['L'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['M'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['N'] = priceMap.get("anjuke_esf_price")
}
}
def map = findAvgPrice(row['C'], cityCodeMap.get(row['D']))
if(map){
row['O'] = map.avg_price
row['P'] = map.credit_value
}
})
}
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
if(d['C']){
def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
if(map){
d['K'] = map.avg_price
d['L'] = map.calc_type
// d['AI'] = map.credit_value
d['M'] = map.month
}
}
})
}
def findThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def findCommunitySummary(id){
def obj = null
def sql = "select * from community_summary where id='${id}'"
ElasticUtils.executeSqlQuery(sql, {
obj = it
})
return obj
}
}
@@ -26,7 +26,7 @@ class MingShenPocTest {
def run(){
def path = "D:\\数据\\民生\\poc测试.xlsx"
//9级标准化
//structureV2(path)//针对海南poc地址数据此接口不合适使用
structureV2(path)//针对海南poc地址数据此接口不合适使用
//省市区拆分
// ner(path)//城市,区域带市带区
// match(path)
@@ -34,7 +34,7 @@ class MingShenPocTest {
// addLatLng(path)
// addAroundPrice(path)
// addAuthor(path)
addHouse(path)
// addHouse(path)
}
def structureV2(path){
@@ -73,6 +73,8 @@ class CommonDatautils {
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -230,6 +232,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
q.put("name", "长滩美墅三期")
DBObject keys = new BasicDBObject();
if(fields){
@@ -781,7 +784,7 @@ class CommonDatautils {
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", new BasicDBObject('$in', regions))
@@ -3418,11 +3421,27 @@ class CommonDatautils {
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def sqmetreRentMap = [:]
for(def d : dbcursor){
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
// sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price)
}
return sqmetreRentMap;
}
static def findAllSqmetreRentPrices(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("cat","district")
q.put("weekend", weekend)
def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000)
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def prices = []
for(def d : dbcursor){
prices << d.avg_price
}
return prices;
}
static def findNewSqmetreRent(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
@@ -3490,5 +3509,14 @@ class CommonDatautils {
}
return obj
}
static def findDistrictAnalyStat(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q)
return analyData
}
}
@@ -82,20 +82,21 @@ class DistrictInfoExportBatch extends BaseBatch{
}
}
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
for(def dist : removeDists){
def key = dist.city +"##"+dist.region+"##"+dist.name
if(!distKeys.contains(key)){
distKeys << key
districts << dist
}
}
// for(def dist : removeDists){
// def key = dist.city +"##"+dist.region+"##"+dist.name
// if(!distKeys.contains(key)){
// distKeys << key
// districts << dist
// }
// }
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
saveOrUpdateFzDist(districts)
// saveOrUpdateFzDist(districts)
remarkRemoveDist(removeDists)
def fzDistricts = getFzDistricts(city)
for(def rMap : fzDistricts){
@@ -185,6 +186,15 @@ class DistrictInfoExportBatch extends BaseBatch{
}
def remarkRemoveDist(removeDists){
for(def rd : removeDists){
def udate = new BasicDBObject()
udate.put("status",1)
udate.put("m_date", new Date())
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
}
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
@@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch {
if(weekendPriceObj.value_type!=1){
if(dist.city == "¸£ÖÝ"){
if(Math.abs(mom)*100>3){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
if(dist.region != "Âíβ"){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
}else{
if(Math.abs(mom)*100>10){
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
task.regionPriceTrendRun()
// task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
@@ -11,26 +12,91 @@ import com.mongodb.DBCollection
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def datas = FileUtils.readLines("D:\\Êý¾Ý\\ÆäËü\\FJW_COMM_20221015.csv")
def rets = []
for(def d : datas){
def arrs = d.split("@")
rets << ["commId":arrs[0], "avg_price":arrs[4]]
}
for(def r : rets){
if(r.avg_price){
def udata = new BasicDBObject()
udata.put("date", 202210)
udata.put("avg", r.avg_price.toInteger())
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
}else{
println r.commId +" ²»´æÔÚ"
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
def city = RegionData.formatCity("余姚")
d['D'] = city
if(city){
d['E'] = RegionData.formatRegion(city, d['C'])
}
// if(!d['D']){
// def citys = CommonDatautils.getCityByRegion(d['A'])
// if(citys && citys.size()==1){
// d['F'] = citys.get(0)
// }
// }
})
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
//def m = [:]
//for(def d : datas){
// m.put(d['B'], d)
//}
//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d->
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(dist){
// def ret = m.get(dist._id.toString())
// if(ret){
// d['E'] = ret['J']
// d['F'] = ret['N']
// }
// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS)
// if(dbo){
// if(dbo.beike_esf_price){
// d['G'] = dbo.beike_esf_price
// }else if(dbo.beike_esf_price){
// d['G'] = dbo.beike_esf_price
// }
//
// if(dbo.maitian_esf_price){
// d['H'] = dbo.maitian_esf_price
// }
// }
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd"))
// if(dbo){
// d['K'] = dbo.avg_price
// d['L'] = dbo.value_type
// }
//
// }
//})
def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", 202304)
return AVERAGE_STATISTICS.findOne(q)
}
//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
//def rets = []
//for(def d : datas){
// def arrs = d.split("@")
// rets << ["commId":arrs[0], "avg_price":arrs[4]]
//}
//
//for(def r : rets){
// if(r.avg_price){
// def udata = new BasicDBObject()
// udata.put("date", 202210)
// udata.put("avg", r.avg_price.toInteger())
// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
// }else{
// println r.commId +" 不存在"
// }
//}
@@ -29,8 +29,11 @@ class AddAvgpriceBatch {
}
def test(){
//发重算均价命令
addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd'))
def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"])
for(def dist : districts){
//发重算均价命令
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
}
}
def run(){
@@ -96,17 +96,28 @@ class CityStatExport {
}
def regionStat(){
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州']
def ret = []
ret << ["城市","区域","小区总量"]
for(def city : citys){
def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){
def row = []
row << city
row << r
def districts = CommonDatautils.findDistricts(city, r, null)
println city +" "+ r +" "+ districts.size()
row << CommonDatautils.queryCityRegionPrice(city, r, null)
def obj = CommonDatautils.findRegionPriceRate(city, r)
row << obj.price_coverage*100
row << districts.size()
ret << row
}
}
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县")
}
@@ -272,6 +272,45 @@ class FiledStrand {
return null
}
static def processName(name){
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
if (name.contains('(') && name.contains(')')){
def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1)
name = name.replace(rep_str, '')
}
if (name.contains('(') && name.contains(')')){
def rep_str = name.subsring(name.index('('),name.rfind(')')+1)
name = name.replace(rep_str, '')
}
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
name = name.replace(rep_str, '')
}
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
name = name.replace(rep_str, '')
}
if (name.find('(')!=-1 && name.find(')')!=-1){
def rep_str = name[name.index('('):name.rfind(')')+1]
name = name.replace(rep_str, '')
}
if (name.find('「')!=-1 && name.find('」')!=-1){
def rep_str = name[name.index('「'):name.rfind('」')+1]
name = name.replace(rep_str, '')
}
return name
}
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
@@ -579,6 +618,12 @@ class FiledStrand {
}else{
return null;
}
}else if(estateType2=="写字楼"){
if(estateCharege>=0.35 && estateCharege<=60){
return estateCharege;
}else{
return null;
}
}
}catch (Exception e){
println "estateCharege ת²»ÁË "+estateCharege
@@ -567,7 +567,7 @@ class OpenApiUtils {
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3"
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
@@ -580,5 +580,79 @@ class OpenApiUtils {
}
//源地址的完整性进行验证
static def addrVerify(city, address){
def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
// 省、市、区、街道、社区居委、POI补全
static def complete(city, address){
def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
//五级(省/市/区/街道/社区居委)解析、补全
static def division(city, address){
def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
//识别地址中省、市、区、道路、POI同时进行地址验证
static def structure(city, address){
def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
/**
* 招商银行测试
* @param city
* @param address
* @return
*/
static def cmbTest(city, address){
def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
}
@@ -0,0 +1,49 @@
package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
if(d['A'] && !d['E']){
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
println d['A'] +" "+d['C']
// def total = findDatas(d['A'], d['B'], d['C'])
// d['D'] = total
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
d['E'] = c
}
})
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name.contains(name) && d.type.contains("住宅")){
total ++
}
})
return total;
}
def findDatas(def city, def region, def name){
def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'"
def total = 0
ElasticUtils.executeSqlQuery(sql, { d->
total = d.count
})
return total;
}
//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
// def region = RegionData.formatCity(d['A'])
// if(!region && d['A']){
// println d['A']
// }
//})
@@ -76,24 +76,24 @@ MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
}
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
//def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
//Calendar calendar = Calendar.getInstance()
//calendar.setTime(startDate)
//def count = 0
//def hashSet = []
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
//
//OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
// def result = JSONObject.fromObject(doc.res_map.result)
// def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
// data << item
// hashSet << result
//}
//
//
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
//ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
@@ -115,4 +115,27 @@ ExcelUtils.write("D:\\
//
//
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230524", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]]
OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode,
doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber]
data << item
hashSet << result
}
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
@@ -0,0 +1,207 @@
package com.fangjia.script.gj.data
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 按区域统计覆盖率
*/
class XiamenStat {
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
public static final def DISTRICT_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
]
static main(args) {
def task = new XiamenStat();
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d->
if(d['D'] && d['D']=="全市"){
def dbo = CommonDatautils.findCityInfo(d['C'].replace("市",""))
if(dbo){
def detail = dbo.district_detail
if(detail){
d['E'] = dbo.existing_district_count//数据量
d['F'] = detail.township*100// 乡镇街道
d['G'] = detail.address*100// 地址
d['H'] = detail.addrAlias*100// 地址别名
d['I'] = detail.name*100// 名称
d['J'] = detail.alias*100// 别名
d['K'] = detail.alias*100// 映射别名
d['L'] = detail.lat2*100// 经纬度(百度)
d['M'] = detail.lat*100// 经纬度(高德)
d['N'] = detail.estateType2*100// 物业类型
d['O'] = detail.buildType*100// 建筑类型
d['P'] = detail.streetCommunity*100// 街道社区
d['Q'] = detail.committee*100// 居委会
d['R'] = detail.businessQuarter*100// 商圈
d['S'] = detail.schools*100// 对口小学
d['T'] = detail.kindergartens*100// 对口幼儿园
d['U'] = detail.loudongCount*100// 楼栋总数
d['V'] = detail.householdCount*100// 户数
d['W'] = detail.estateCharge*100// 物业费
d['X'] = detail.carSiteCount*100// 停车位
d['Y'] = ""// 电梯
d['Z'] = detail.propertyLife*100// 产权年限
d['AA'] = detail.houseProperty*100// 产权性质
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
d['AC'] = detail.landProperty*100// 小区性质
d['AD'] = detail.finishBuildingDate*100// 竣工时间
d['AE'] = detail.openDate*100// 交房时间
d['AF'] = detail.allArea*100// 占地面积
d['AG'] = detail.buildArea*100// 建筑面积
d['AH'] = detail.cubagePercent*100// 容积率
d['AI'] = detail.virescencePercent*100// 绿化率
d['AJ'] = detail.mainType*100// 主力户型
d['AK'] = detail.mainTypeArea*100// 主力户型面积
d['AL'] = detail.developer*100// 开发商
d['AM'] = detail.estateCompany*100// 物业公司
d['AN'] = dbo.price_coverage*100// 小区当前均价
}
}
}else{
def fields = DISTRICT_FIELDS_MAP.keySet().asList()
def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields)
def detail = districtDetail(regionDistricts.size(), regionDistricts);
if(detail){
d['E'] = regionDistricts.size()//数据量
d['F'] = detail.township*100// 乡镇街道
d['G'] = detail.address*100// 地址
d['H'] = detail.addrAlias*100// 地址别名
d['I'] = detail.name*100// 名称
d['J'] = detail.alias*100// 别名
d['K'] = detail.alias*100// 映射别名
d['L'] = detail.lat2*100// 经纬度(百度)
d['M'] = detail.lat*100// 经纬度(高德)
d['N'] = detail.estateType2*100// 物业类型
d['O'] = detail.buildType*100// 建筑类型
d['P'] = detail.streetCommunity*100// 街道社区
d['Q'] = detail.committee*100// 居委会
d['R'] = detail.businessQuarter*100// 商圈
d['S'] = detail.schools*100// 对口小学
d['T'] = detail.kindergartens*100// 对口幼儿园
d['U'] = detail.loudongCount*100// 楼栋总数
d['V'] = detail.householdCount*100// 户数
d['W'] = detail.estateCharge*100// 物业费
d['X'] = detail.carSiteCount*100// 停车位
d['Y'] = ""// 电梯
d['Z'] = detail.propertyLife*100// 产权年限
d['AA'] = detail.houseProperty*100// 产权性质
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
d['AC'] = detail.landProperty*100 // 小区性质
d['AD'] = detail.finishBuildingDate*100// 竣工时间
d['AE'] = detail.openDate*100// 交房时间
d['AF'] = detail.allArea*100// 占地面积
d['AG'] = detail.buildArea*100// 建筑面积
d['AH'] = detail.cubagePercent*100// 容积率
d['AI'] = detail.virescencePercent*100// 绿化率
d['AJ'] = detail.mainType*100// 主力户型
d['AK'] = detail.mainTypeArea*100// 主力户型面积
d['AL'] = detail.developer*100// 开发商
d['AM'] = detail.estateCompany*100// 物业公司
// d['AN'] = ""// 小区当前均价
}
}
})
}
def districtDetail(def disCount, def regionDistricts) {
def countMap = [:]
for(def dbo : regionDistricts) {
DISTRICT_FIELDS_MAP.each() {key,skey->
if (isValid(dbo.get(key))) {
if(countMap.containsKey(key)) {
countMap.put(key, countMap.get(key) + 1)
}else {
countMap.put(key, 1)
}
}
}
}
def districtDetail = [:];
DISTRICT_FIELDS_MAP.each() {key,skey->
if(countMap.containsKey(key)) {
districtDetail.put(skey, roundDouble(countMap.get(key), disCount))
}else {
districtDetail.put(skey, 0)
}
}
if(countMap.containsKey("lat") && countMap.containsKey("lng")) {
districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount));
}else {
districtDetail.put('latLng', 0);
}
if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) {
districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount));
}else {
districtDetail.put('latLng2', 0);
}
return districtDetail;
}
def CONTAINS_KWS = ["...", "暂无", "待定", "未知"]
def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."]
def isValid(def value) {
if (!value) {
return false;
}
if (value instanceof String) {
value = value.trim()
for(String ck : CONTAINS_KWS) {
if(value.contains(ck)) {
return false;
}
}
for(String ek : EQUALS_KWS) {
if(ek.equals(value)) {
return false;
}
}
} else if (value instanceof Number) {
if (value <= 0) {
return false;
}
} else if (value instanceof Collection || value instanceof Map) {
if (value.size() == 0) {
return false;
}
}
return true;
}
double roundDouble(int a, int base) {
if(a == 0 || base == 0) {
return 0
}
return ((double)a / (double)base * 10000.0).toInteger() / 10000.0
}
}
@@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = ["ÕÄÖÝ"]//CommonDatautils.findAllCity().reverse()
def citys = CommonDatautils.findAllCity().reverse()
def analysRets = []
@@ -95,7 +95,7 @@ class FangjiaClientUtils {
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -0,0 +1,79 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 写字楼字段清洗入库(自动入库)
*/
@BatchOwner(Watcher.GaoJuan)
class OfficeFieldCleanBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new OfficeFieldCleanBatch(), args)
}
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
for(def c : citys){
def offices = findAllOffice(c)
strandField(offices)
}
}
def strandField(offices){
def fields = ["property_fee","build_area","","","","","","","","","",""]
for(def office : offices){
fields.each{
def newVal = null
if(it == "property_fee"){
newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼")
newVal = newVal.toString()
}
if(it =="all_area" || it =="build_area"){
newVal = FiledStrand.processArea(office.get(it))
if(newVal && newVal > 5000){
newVal = newVal.toString()
}
}
if(it =="name"){
newVal = FiledStrand.processName(office.get(it))
if(newVal){
newVal = newVal.toString()
}
}
}
}
}
def findAllOffice(city){
def sql = "select * from office where city='${city}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
})
return rets
}
}
@@ -144,5 +144,8 @@ class SeaweedDataMatchProcess {
}
public static void main(def agrs){
doMatch("上海", "静安", "金家巷西弄", null)
}
}
@@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&
stat()
def stat(){
def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1)
def map=[:]
for(def d : datas){
def count = map.get(d['C'])
@@ -69,8 +69,8 @@ def findLogs(){
def findFangjiaLogs(){
def startDate = DateUtil.parseDate("20230329", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230429", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230531", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def list = []
@@ -84,14 +84,21 @@ def findFangjiaLogs(){
QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
// def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def q = QueryBuilders.fuzzyQuery("request", "/enquire")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
if (houses) {
for (def d : houses) {
def reqStr = d.request.toString()
if(!reqStr.contains("/enquire")){
continue
}
def mid = null
if (reqStr.contains("mid=")) {
@@ -107,5 +114,5 @@ def findFangjiaLogs(){
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志")
ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志")
}
@@ -0,0 +1,232 @@
package com.fangjia.script.gj.es
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils
import org.apache.lucene.queryparser.classic.QueryParser
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class HouseAnaly {
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new HouseAnaly()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def allRets = []
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"])
def rets = processing(districts)
println "[${city}] 写文件 "+rets.size()
allRets.addAll(rets)
}
ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析")
}
def processing(dists){
def rets = new CopyOnWriteArrayList()
rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(dist.alias && dist.alias.size()>=1){
def ret = analy(dist)
if(ret){
rets.addAll(ret)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def analy(dist){
def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是");
def alias = dist.alias
for(def aliasName : alias){
if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){
def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否");
if(aliaRet){
masterRet.addAll(aliaRet)
}
}
}
if(masterRet.size()>=2){
def masterAvgPrice = 0
def masterMidPrice = 0
def back = false
for(def row : masterRet){
if(row.get(4)=="是"){
masterAvgPrice = row.get(8)
masterMidPrice = row.get(9)
}else{
def pre1 = 0
def pre2 = 0
if(masterAvgPrice>0){
pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice)
}
if(masterMidPrice>0){
pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice)
}
if(pre1>=20 || pre2>=20){
back = true
}
}
}
if(back){
return masterRet
}
}
return null;
}
//小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量
def runAnalying(id, city, region, name, flag){
def houses = findHouse(city, region, name);
def rets = []
if(houses && houses.size()>=5){
def prices = []
for(def house : houses){
prices << house.avg_price
}
prices = prices.sort()
def average = (int)(prices.sum() / prices.size())
def midPrice = getMidPrice(prices)
def price10 = null
def price90 = null
if(houses.size()>=10){
def postion = Math.round(houses.size()/10)
price10 = prices.get((int)(postion-1))
price90 = prices.get((int)(houses.size()-postion-1))
}
def countMap = classType(houses)
// def pre = java.lang.Math.abs((midPrice-average)*100/average)
// if(pre>=20){
rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")]
// }
}
return rets
}
def classType(houses){
def district = 0
def villa = 0
def business = 0
for(def h : houses){
if(h.estate_type){
if(h.estate_type == "别墅"){
villa = villa + 1
continue
}else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){
business = business + 1
continue
}
}
def valla = isVilla(h.area, h.name, h.height, h.room)
if(valla){
villa = villa + 1
continue
}
district = district + 1
}
return ["district":district, "villa":villa, "business":business]
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getMidPrice(def prices){
def mid = prices.size()/2
if(mid > (int)mid){
return prices.get((int)mid)
}else{
def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1))
return (int)(total/2)
}
return null
}
def findHouse(def city, def region, def name){
def houses = []
try{
name = escapeExprSpecialWord(name)
String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'"
ElasticUtils.executeSqlQuery(sql,{ d->
def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type]
houses << r
// println r
})
}catch(Exception e){
e.printStackTrace()
}
return houses
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
@@ -21,13 +21,23 @@ class GaoDeMapSearchTest {
static main(args) {
def task = new GaoDeMapSearchTest();
def path = "D:\\数据\\光大\\光大测试-V2.xlsx"
def path = "D:\\数据\\富民\\富民测试500.xlsx"
// task.structureV2(path)
// task.match(path)
// task.mapSearch(path)
task.baiduMapSearch(path)
task.checkTrue(path)
// task.baiduMapSearch(path)
// task.checkTrue(path)
task.addFinancialLevel(path)
}
def addFinancialLevel(path){
ExcelUtils.multiThreadingModify(path, { row ->
def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A'])
if(results){
row['V'] = results.consumptionRating
}
});
}
def checkTrue(path){
@@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
//广州,佛山 重庆
static main(args) {
def task = new BeiKeCrawlByChrome();
@@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome {
println city +" 已运行"
continue
}
def newCity = RegionData.formatCity(city)
if(!newCity){
if(city=="昆山"){
newCity = "苏州"
}
println city +" 无法格式化"
}
def url = m[1]
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
@@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome {
def href = url+ r.attr("href")
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
if(city == "昆山"){
regionText = "昆山"
}
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println url +" 详情链接:"+detailUrls.size()
}
}
@@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, region)
def dbo = findTrade(durl , "贝壳找房")
if (!dbo) {
parse(city, e, region, driver)
}
}else{
stop = true;
break;
@@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome {
return detailUrls
}
def parse(city,e, region){
def parse(city,e, region, driver){
try {
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
if(title.contains("车位")){
detail.type = "车位"
}
detail.city = city
detail.district_name = arrs[0]
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
@@ -266,28 +289,42 @@ public class BeiKeCrawlByChrome {
}
detail.source = "贝壳找房"
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡",""))
if(detail.total_price && detail.avg_price){
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
detail.url = detailUrl
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
detail.floor_info = floorStr
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorInfo && floorInfo.isNumber()) {
detail.height = floorInfo.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}catch(Exception ex){
@@ -392,16 +429,16 @@ public class BeiKeCrawlByChrome {
}
def findTrade(def detail){
def findTrade(def url, source){
def q = new BasicDBObject()
// q.put("city", detail.city)
// q.put("region", detail.region)
// q.put("district_name", detail.district_name)
// q.put("area", detail.area)
// q.put("total_price", detail.total_price)
q.put("url", detail.url)
q.put("url", url)
// q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
q.put("source", source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
@@ -446,12 +483,15 @@ public class BeiKeCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
Thread.sleep(10000)
Thread.sleep(20000)
}catch(Exception e){
Thread.sleep(10000)
Thread.sleep(15000)
}
return driver
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.paimai
class DataFormatUtils {
static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000]
static def parseLift(def str){
if(str && str.contains("梯")){
def lift = str.toString().substring(0, str.toString().indexOf("梯"))
if(lift){
def val = chineseIntegerPart2Digit(lift)
if(val){
return (int)val
}
}
}
return null
}
static def parseHousehold(def str){
if(str && str.contains("梯") && str.contains("户")){
def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户"))
def val = chineseIntegerPart2Digit(household)
if(val){
return (int)val
}
}
return null
}
static def chineseIntegerPart2Digit(String chineseDigit)
{
Double finalDigit = 0.0;
Integer r = 1;
for(int i = chineseDigit.length()-1; i>=0;i--)
{
Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i)));
if( val >= 10 && i == 0 )
{
if (val > r)
{
r = val;
finalDigit = finalDigit +val;
}
else
{
r = r * val;
}
}
else if(val >= 10)
{
if (val > r)
{
r = val;
}
else
{
r = r * val;
}
}
else
{
finalDigit = finalDigit + r*val;
}
}
return finalDigit;
}
static String getValue(String s) {
if (s && s.contains("暂无")) {
return null
}
return s
}
}
@@ -196,9 +196,22 @@ public class LianjiaCrawlByChrome {
println city +" 已运行"
continue
}
def newCity = RegionData.formatCity(city)
if(!newCity){
if(city=="昆山"){
newCity = "苏州"
}
println city +" 无法格式化"
}
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
driver.get(newUrl)
try{
driver.get(newUrl)
}catch(Exception e){
println newUrl +" 链接有问题!"
continue
}
//def detailUrls = getPageUrls(driver, cityUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
@@ -208,9 +221,11 @@ public class LianjiaCrawlByChrome {
for(def r : regions){
def regionText = r.text()
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
if(city == "昆山"){
regionText = "昆山"
}
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
}
}
@@ -284,7 +299,10 @@ public class LianjiaCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver)
}
}else{
stop = true;
break;
@@ -302,85 +320,106 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e, regionText){
def parse(city,e, regionText, driver){
try {
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
def price = e.select(".totalPrice .number").text()
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
if(title.contains("车位")){
detail.type = "车位"
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
detail.city = city
detail.district_name = arrs[0]
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
}
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
}
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = (int)(price.toDouble() * 10000)
}
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
if(detail.total_price && detail.avg_price){
def dbo = findTrade(detail)
if (!dbo) {
if (price && price.isNumber()) {
detail.total_price = (int)(price.toDouble() * 10000)
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = detailUrl
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层",""))
if(detail.total_price && detail.avg_price){
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
detail.floor_info = floorStr
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr2 && floorStr2.isNumber()) {
detail.height = floorStr2.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}
}catch(Exception ex){
@@ -484,16 +523,16 @@ public class LianjiaCrawlByChrome {
}
def findTrade(def detail){
def findTrade(def url, source){
def q = new BasicDBObject()
// q.put("city", detail.city)
// q.put("region", detail.region)
// q.put("district_name", detail.district_name)
// q.put("area", detail.area)
// q.put("total_price", detail.total_price)
q.put("url", url)
// q.put("trade_date", detail.trade_date)
q.put("url", detail.url)
q.put("source", detail.source)
q.put("source", source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
@@ -535,7 +574,7 @@ public class LianjiaCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
Thread.sleep(10000)
Thread.sleep(15000)
}catch(Exception e){
Thread.sleep(10000)
}
@@ -0,0 +1,36 @@
package com.fangjia.script.gj.region
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 格式化其它网站区域
*/
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl)
def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1)
for(def d : datas){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("recognizeCity", d['F'])
q.put("recognizeRegion", d['A'])
def count = SERVER_AREA_COLL.count(q)
if(!count){
def obj = new BasicDBObject()
obj.put("city", d['A'])
obj.put("recognizeRegion", d['A'])
obj.put("recognizeCity", d['F'])
obj.put("createdDate", new Date())
SERVER_AREA_COLL.save(obj)
}else{
println d['A']+" "+ d['F'] +" 已存在"
}
}
@@ -237,7 +237,7 @@ public class DbUtils{
def lng = district.lng / 10000000000.0
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0));
query.append('$sd', sdq)
def keys = new BasicDBObject()