Merge remote-tracking branch 'origin/script_gj' into script_gj
This commit is contained in:
+95
-7
@@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ServerAreaMap
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* Created by dxy on 2016/12/15.
|
||||
*/
|
||||
@@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch {
|
||||
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def DISTRICT_STAT_MAP = [:]
|
||||
|
||||
public void process(Object args) {
|
||||
def cities = ServerAreaMap.findAllCities()
|
||||
|
||||
|
||||
def cities = ["深圳"]//ServerAreaMap.findAllCities()
|
||||
for(String city : cities) {
|
||||
DISTRICT_STAT_MAP.clear()
|
||||
Map<String, List<String>> regions = RegionData.getAllRegions(city)
|
||||
regions.each {region, alias->
|
||||
processRegionRank(city, region)
|
||||
}
|
||||
processCityRank(city)
|
||||
// DISTRICT_STAT_MAP.clear()
|
||||
// Map<String, List<String>> regions = RegionData.getAllRegions(city)
|
||||
// regions.each {region, alias->
|
||||
// processRegionRank(city, region)
|
||||
// }
|
||||
// processCityRank(city)
|
||||
|
||||
processCityRentRank(city)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch {
|
||||
return ret
|
||||
}
|
||||
|
||||
def processCityRentRank(city){
|
||||
def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd")
|
||||
def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth)
|
||||
def prices = []
|
||||
distRentsMap.each{key, price->
|
||||
if(!prices.contains(price)){
|
||||
prices << price
|
||||
}
|
||||
}
|
||||
// prices = prices.unique()
|
||||
prices = prices.sort().reverse()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def keys = distRentsMap.keySet().asList()
|
||||
final CountDownLatch latch = new CountDownLatch(keys.size())
|
||||
|
||||
// for(def i=0;i<keys.size();i++){
|
||||
// final def key = keys[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
// totalCount.addAndGet(1)
|
||||
// upateDistrict(key, distRentsMap, prices)
|
||||
// println totalCount.get()+" "+ key
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
// latch.await()
|
||||
|
||||
def index = 0
|
||||
distRentsMap.each{key, price->
|
||||
index ++
|
||||
def arrs = key.split("##");
|
||||
double percentile = calculatePercentile(prices, price);
|
||||
|
||||
DBObject q = new BasicDBObject("city", arrs[0])
|
||||
q.append("region", arrs[1])
|
||||
q.append("district_name", arrs[2])
|
||||
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("rent_city_rank", (int)percentile);
|
||||
udata.put("m_date", new Date())
|
||||
DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
println index +" "+ arrs[2]
|
||||
}
|
||||
}
|
||||
|
||||
def upateDistrict(key, distRentsMap, prices){
|
||||
def arrs = key.split("##");
|
||||
def price = distRentsMap.get(key)
|
||||
|
||||
double percentile = calculatePercentile(prices, price);
|
||||
|
||||
DBObject q = new BasicDBObject("city", arrs[0])
|
||||
q.append("region", arrs[1])
|
||||
q.append("district_name", arrs[2])
|
||||
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("rent_city_rank", (int)percentile);
|
||||
udata.put("m_date", new Date())
|
||||
// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
}
|
||||
|
||||
public static int calculatePercentile(def values, def targetValue) {
|
||||
// 找出数字在List中的排名
|
||||
def rank = values.indexOf(targetValue) + 1
|
||||
// 计算百分位数
|
||||
def percentile = (int)((rank / values.size()) * 100)
|
||||
return percentile;
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictStatRankBatch(), args)
|
||||
|
||||
+21
-3
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -51,7 +51,19 @@ for(def d : datas){
|
||||
map.region = d['B']
|
||||
def roomStr = d['E']
|
||||
if(roomStr){
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
if(roomStr.contains("-")){
|
||||
def arrs = roomStr.split("-")
|
||||
try{
|
||||
map.room = arrs[0].toString().toInteger()
|
||||
}catch(Exception e){
|
||||
println "room error"
|
||||
}
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
}else{
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
@@ -59,7 +71,13 @@ for(def d : datas){
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
def trade_date = d['J']
|
||||
map.trade_date = trade_date
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
|
||||
-1
@@ -77,7 +77,6 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
|
||||
+4
-3
@@ -66,7 +66,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
println "curDate:"+curDate
|
||||
// for(def city : citys){
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
@@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(trades.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
@@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
}
|
||||
|
||||
+1
-1
@@ -428,7 +428,7 @@ class CcbCommonUtils {
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
|
||||
+41
-3
@@ -28,9 +28,11 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
task.deleteThirdPartyData()
|
||||
// task.deleteThirdPartyData()//删除第三方
|
||||
// task.delDealByPe()
|
||||
// task.delDealData()
|
||||
|
||||
task.exportThirdParty()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -372,15 +374,29 @@ class DealPriceDataProcess {
|
||||
return map
|
||||
}
|
||||
|
||||
def exportThirdParty(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def results = []
|
||||
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty3(city, 202305)
|
||||
for(def map : rets){
|
||||
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
|
||||
}
|
||||
println city +" "+ rets.size()
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方")
|
||||
}
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202303
|
||||
def month = 202304
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202304)
|
||||
def rets = findPeGteThirdParty2(city, 202305)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
@@ -431,6 +447,28 @@ class DealPriceDataProcess {
|
||||
return rets
|
||||
}
|
||||
|
||||
//城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE
|
||||
|
||||
def findPeGteThirdParty3(cityName, yearmonth){
|
||||
def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15"
|
||||
def rets = []
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityid = r.urbn_id
|
||||
map.city = r.urbn_nm
|
||||
map.data_id = r.gujia_fsid
|
||||
map.region = r.gujia_rgon_nm
|
||||
map.comm_id = r.gujia_comm_id
|
||||
map.district_name = r.gujia_comm_nm
|
||||
map.type = r.gujia_bds_clc_tp
|
||||
map.clc_price = r.gujia_bds_avprc
|
||||
map.price = r.gujiabds_to_mkprc_mny
|
||||
map.ppe = r.gujia_mns_rto
|
||||
rets << map
|
||||
});
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def delDealData(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d ->
|
||||
|
||||
+3
-3
@@ -79,9 +79,9 @@ class ExportOracleAllDataTest {
|
||||
// exportCommCaseDetailGj()//
|
||||
|
||||
// // ok文件
|
||||
exportInOk()
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
|
||||
// exportInOk()
|
||||
// SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
|
||||
|
||||
}
|
||||
|
||||
|
||||
+12
-10
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
// task.updatePrice()
|
||||
// task.addFjwPrice()
|
||||
task.addFjwPrice()
|
||||
// task.addCcbPrice()//oracle
|
||||
|
||||
// task.addCcbPriceFromGuided()
|
||||
@@ -42,7 +42,7 @@ class FjwPricePkCcbPrice {
|
||||
// task.exportFjwPriceAndCcbPrice()
|
||||
// task.addNewDistPrice()
|
||||
|
||||
task.addCcbPrice2()//按id查
|
||||
// task.addCcbPrice2()//按id查
|
||||
// task.addCcbPrice3()//按city,region,name查
|
||||
// task.standRegion()
|
||||
// task.addGuidPrice()
|
||||
@@ -217,17 +217,19 @@ class FjwPricePkCcbPrice {
|
||||
}
|
||||
|
||||
def addFjwPrice(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d->
|
||||
if(!d['N']){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(dist){
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
d['K'] = dbo.avg_price
|
||||
}
|
||||
d['N'] = dist.region
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd"))
|
||||
// if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -83,8 +83,8 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
|
||||
//更新静态数据表
|
||||
//
|
||||
// //更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
+34
-34
@@ -33,8 +33,8 @@ import java.util.regex.*;
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
@@ -1287,18 +1287,18 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['B'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
//// if(dist){
|
||||
//// d['E'] = dist.region
|
||||
//// d['F'] = dist._id.toString()
|
||||
//// }
|
||||
//// }
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
if(region){
|
||||
def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
if(dist){
|
||||
d['G'] = dist.region
|
||||
d['H'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
// if(!d['F']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
////
|
||||
@@ -1308,30 +1308,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
// d['F'] = dist.status
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
if(count && count==1){
|
||||
def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
d['E'] = dist.city
|
||||
}
|
||||
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['J'] = dist.status
|
||||
// }
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// d['F'] = newRegion
|
||||
},0)
|
||||
|
||||
|
||||
|
||||
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
// if(count && count==1){
|
||||
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
// d['E'] = dist.city
|
||||
// }
|
||||
//
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['J'] = dist.status
|
||||
//// }
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
//// d['F'] = newRegion
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify(fileName, {d->
|
||||
// if(d['A']){
|
||||
// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'"
|
||||
|
||||
+1
-1
@@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
|
||||
|
||||
+6
-5
@@ -45,6 +45,7 @@ class TradeStatLog {
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
|
||||
// task.deleteDistrict()
|
||||
// task.deleteDealData()
|
||||
// task.deleteDealData2()
|
||||
@@ -92,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-02-01"
|
||||
def date = "2023-03-01"
|
||||
def endDate = "2023-04-01"
|
||||
def newDate = "2023-05-01"
|
||||
def startDate = "2023-03-01"
|
||||
def date = "2023-04-01"
|
||||
def endDate = "2023-05-01"
|
||||
def newDate = "2023-06-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+2
-2
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
|
||||
cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
// cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
// regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
|
||||
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
class FumingTest {
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
def task = new FumingTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run() {
|
||||
def path = "D:\\数据\\富民\\富民测试500-v7.xlsx"
|
||||
// match(path)
|
||||
// addPrice(path)
|
||||
addThirdPrice(path)
|
||||
// addLastPrice(path)
|
||||
// addFacLevel(path)
|
||||
}
|
||||
|
||||
|
||||
def addFacLevel(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['K'] && row['I']){
|
||||
// def obj = findCommunitySummary(row['C'])
|
||||
// if(obj){
|
||||
// row['K'] = obj.consumption_rating
|
||||
// }else{
|
||||
row['K'] = getNearByDistrictLevel(row['C'], row['I'])
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 一公里小区数量
|
||||
*/
|
||||
def getNearByDistrictLevel(def id, def price){
|
||||
BasicDBObject district = DbUtils.seaweed.findById(id);
|
||||
def level = null
|
||||
if(district){
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(district);
|
||||
println id +" 共:"+list.size()+" "+ price
|
||||
if(list!=null&&list.size()>0){
|
||||
for(def obj : list){
|
||||
def summary = findCommunitySummary(obj._id.toString())
|
||||
if(summary && summary.avg_price && summary.consumption_rating){
|
||||
def pre = java.lang.Math.abs((summary.avg_price - price)*100/price)
|
||||
println summary.name +" "+pre +" "+ summary.avg_price
|
||||
if(pre <= 10){
|
||||
level = summary.consumption_rating
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return level;
|
||||
}
|
||||
|
||||
|
||||
def addLastPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Z'] ){
|
||||
if(row['W']){
|
||||
row['Z'] = row['W']
|
||||
}else if(row['R']){
|
||||
row['Z'] = row['R']
|
||||
}else if(row['I']){
|
||||
row['Z'] = row['I']
|
||||
}
|
||||
|
||||
if(row['Z']){
|
||||
row['J'] = (int)(row['Z']*row['B'].toString().toDouble())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['N'] ){
|
||||
def city = row['C']
|
||||
def address = row['A']
|
||||
def region = row['D']
|
||||
def name = row['G']
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['N'] = jsonObject.get("id")
|
||||
row['O'] = jsonObject.get("city")
|
||||
row['P'] = jsonObject.get("district")
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['R'] = jsonObject.get("propertyType")
|
||||
row['S'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['C']
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
row['D'] = dist.city
|
||||
row['E'] = dist.region
|
||||
row['F'] = dist.name
|
||||
row['G'] = dist.estate_type2
|
||||
row['H'] = dist.address
|
||||
// row['Q'] = dist.name
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['I'] = priceObj.avg_price
|
||||
row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble())
|
||||
row['R'] = priceObj.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连']
|
||||
def allMap = [:]
|
||||
// for(def c : citys){
|
||||
// def cityMap = findThirdPartyPrice(c)
|
||||
// allMap << cityMap
|
||||
// }
|
||||
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
|
||||
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['L'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("beike_esf_price")){
|
||||
row['M'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("anjuke_esf_price")){
|
||||
row['N'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
def map = findAvgPrice(row['C'], cityCodeMap.get(row['D']))
|
||||
if(map){
|
||||
row['O'] = map.avg_price
|
||||
row['P'] = map.credit_value
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addCcbPrice2(){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
|
||||
if(d['C']){
|
||||
def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
|
||||
if(map){
|
||||
d['K'] = map.avg_price
|
||||
d['L'] = map.calc_type
|
||||
// d['AI'] = map.credit_value
|
||||
d['M'] = map.month
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def findCommunitySummary(id){
|
||||
def obj = null
|
||||
def sql = "select * from community_summary where id='${id}'"
|
||||
ElasticUtils.executeSqlQuery(sql, {
|
||||
obj = it
|
||||
})
|
||||
return obj
|
||||
}
|
||||
}
|
||||
+70
-6
@@ -40,19 +40,25 @@ class HailanTest {
|
||||
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new HailanTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\海南\\统计.xlsx"
|
||||
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
|
||||
// addThirdPrice(inputfile)
|
||||
// match(inputfile)
|
||||
// tradeAvgPrice(inputfile)
|
||||
// caluateTotalPrice(inputfile)
|
||||
// addPrice(inputfile)
|
||||
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
|
||||
// addHouse(inputfile)
|
||||
// stat(inputfile)
|
||||
hailanStat(inputfile)
|
||||
// hailanStat(inputfile)
|
||||
addFzFjwPrice(inputfile)
|
||||
}
|
||||
|
||||
def addHouse(inputfile){
|
||||
@@ -533,13 +539,13 @@ class HailanTest {
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['C']){
|
||||
String city = row['H']//RegionData.formatCity(row['B'])
|
||||
if(row['A']){
|
||||
String city = "福州"//RegionData.formatCity(row['B'])
|
||||
if(city){
|
||||
def address = row['C']
|
||||
def address = row['A']
|
||||
// def region = row['C']
|
||||
println "[match] city:"+city+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E'])
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['I'] = jsonObject.get("district")
|
||||
row['J'] = jsonObject.get("name")
|
||||
@@ -554,6 +560,8 @@ class HailanTest {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def stat(path){
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['L'] && !row['H']){
|
||||
@@ -700,4 +708,60 @@ class HailanTest {
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
def tradeAvgPrice(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){
|
||||
def dist = CommonDatautils.findDistrictById(row['I'])
|
||||
if(dist){
|
||||
def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01");
|
||||
row['J'] = (int)tradeAvgPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate)
|
||||
def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def caluateTotalPrice(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(row['J']){
|
||||
row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble())
|
||||
}else if(row['K']){
|
||||
row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addFzFjwPrice(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['B']) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "福州")
|
||||
q.put("region", row['A'])
|
||||
q.put("name", row['B'])
|
||||
q.put("month", 202304)
|
||||
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
||||
if (obj) {
|
||||
row['D'] = obj.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -26,7 +26,7 @@ class MingShenPocTest {
|
||||
def run(){
|
||||
def path = "D:\\数据\\民生\\poc测试.xlsx"
|
||||
//9级标准化
|
||||
//structureV2(path)//针对海南poc地址数据此接口不合适使用
|
||||
structureV2(path)//针对海南poc地址数据此接口不合适使用
|
||||
//省市区拆分
|
||||
// ner(path)//城市,区域带市带区
|
||||
// match(path)
|
||||
@@ -34,7 +34,7 @@ class MingShenPocTest {
|
||||
// addLatLng(path)
|
||||
// addAroundPrice(path)
|
||||
// addAuthor(path)
|
||||
addHouse(path)
|
||||
// addHouse(path)
|
||||
}
|
||||
|
||||
def structureV2(path){
|
||||
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
class RuiFenPocTest {
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
def task = new RuiFenPocTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\瑞丰\\房产信息-地址V3-1.xlsx"
|
||||
// match(inputfile)
|
||||
// addValue(inputfile)
|
||||
|
||||
// addThirdPrice(inputfile)
|
||||
lastPrice(inputfile)
|
||||
}
|
||||
|
||||
|
||||
def lastPrice(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(!row['X'] && row['S']){
|
||||
def mod = java.lang.Math.abs((row['S'].toString().toDouble()-row['H'].toString().toDouble())*100/row['H'].toString().toDouble())
|
||||
if(mod <= 10){
|
||||
row['U'] = row['H']
|
||||
}else{
|
||||
row['U'] = row['S']
|
||||
}
|
||||
}else{
|
||||
if(row['K']){
|
||||
row['U'] = row['K']
|
||||
}else{
|
||||
row['U'] = row['H']
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addThirdPrice(path){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
|
||||
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['L'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("beike_esf_price")){
|
||||
row['M'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("anjuke_esf_price")){
|
||||
row['N'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
def map = findAvgPrice(row['I'], cityCodeMap.get(row['D']))
|
||||
if(map){
|
||||
row['O'] = map.avg_price
|
||||
row['P'] = map.credit_value
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addValue(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
def dist = CommonDatautils.findDistrictById(row['I'])
|
||||
if(dist){
|
||||
row["D"] = dist.city
|
||||
row["E"] = dist.region
|
||||
row["F"] = dist.name
|
||||
row["G"] = dist.address
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['H'] = priceObj.avg_price
|
||||
row['K'] = priceObj.avg_price_2b
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def match(path) {
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def city = "绍兴"
|
||||
if (row['B'] && !row["D"]) {
|
||||
def address = row['B']
|
||||
println "[match] city:" + city + " address:" + address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null)
|
||||
|
||||
if(!row["D"]){
|
||||
if (jsonObject != null && jsonObject.precise != -1) {
|
||||
row["D"] = jsonObject.city
|
||||
row["E"] = jsonObject.district
|
||||
row["F"] = jsonObject.name
|
||||
row["G"] = jsonObject.address
|
||||
row["I"] = jsonObject.id
|
||||
def priceObj = CommonDatautils.findDistrictPrice(jsonObject.city, jsonObject.district, jsonObject.name)
|
||||
if(priceObj){
|
||||
row['H'] = priceObj.avg_price
|
||||
}
|
||||
}
|
||||
}else {
|
||||
if (jsonObject != null && jsonObject.precise != -1) {
|
||||
if(row['J']){
|
||||
def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id
|
||||
row['K'] = str
|
||||
}else{
|
||||
def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id
|
||||
row['J'] = str
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+412
-5
@@ -1,11 +1,14 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
@@ -23,6 +26,12 @@ import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CommonDatautils {
|
||||
|
||||
@@ -73,6 +82,19 @@ class CommonDatautils {
|
||||
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
|
||||
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -230,6 +252,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "长滩美墅三期")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -781,7 +804,7 @@ class CommonDatautils {
|
||||
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", new BasicDBObject('$in', regions))
|
||||
@@ -1001,7 +1024,7 @@ class CommonDatautils {
|
||||
q.put("cat", cat);
|
||||
q.put("region", region)
|
||||
q.put("name", name);
|
||||
q.put("status", 0)
|
||||
q.put("visible", 0)
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -3418,11 +3441,27 @@ class CommonDatautils {
|
||||
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def sqmetreRentMap = [:]
|
||||
for(def d : dbcursor){
|
||||
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
|
||||
// sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
|
||||
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price)
|
||||
}
|
||||
return sqmetreRentMap;
|
||||
}
|
||||
|
||||
|
||||
static def findAllSqmetreRentPrices(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("cat","district")
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000)
|
||||
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def prices = []
|
||||
for(def d : dbcursor){
|
||||
prices << d.avg_price
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
static def findNewSqmetreRent(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
@@ -3490,5 +3529,373 @@ class CommonDatautils {
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
static def findDistrictAnalyStat(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q)
|
||||
return analyData
|
||||
}
|
||||
|
||||
static def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
|
||||
.must(date_range).must(area_range)
|
||||
if(cat){
|
||||
queryBuilder.must(QueryBuilders.termQuery("cat", cat))
|
||||
}
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(5);
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
// println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
//// println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
// continue;
|
||||
// }
|
||||
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
// if(vallaFlag){
|
||||
// continue;
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.area = seawater.area
|
||||
rowMap.address = seawater.address
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.build_type = seawater.build_type
|
||||
rowMap.build_date = seawater.build_date
|
||||
rowMap.district_name = seawater.district_name
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public boolean isVilla(area, name, height, room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 根据经纬度获取区域
|
||||
* @param city
|
||||
* @param location
|
||||
* @return
|
||||
*/
|
||||
static def getRegionByLocation(city, def location){
|
||||
def json = GaoDeApiUtils.geocode(location);
|
||||
def region = null;
|
||||
if (json && json.info=="OK") {
|
||||
def regeocode = json.regeocode
|
||||
if(regeocode && regeocode.addressComponent){
|
||||
region = regeocode.addressComponent.district
|
||||
if(!region){
|
||||
region = regeocode.addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
if(region){
|
||||
region = RegionData.formatRegion(city, region)
|
||||
}
|
||||
return region;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package com.fangjia.script.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class MasterNameInAlias {
|
||||
|
||||
static main(args) {
|
||||
def task = new MasterNameInAlias()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allList = []
|
||||
for(def city : citys){
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
|
||||
|
||||
def cityList = []
|
||||
cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"]
|
||||
for(def dist1 : disticts){
|
||||
def distList = []
|
||||
for(def dist2 : disticts){
|
||||
if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){
|
||||
if(dist2.alias && dist2.alias.contains(dist1.name)){
|
||||
def distance = ''
|
||||
if(dist1.lng && dist2.lng){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d);
|
||||
}
|
||||
distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance]
|
||||
dist2.alias.remove(dist1.name)
|
||||
def d = new BasicDBObject()
|
||||
d.put("_id", dist2._id)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", dist2.alias)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
if(distList){
|
||||
cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', '']
|
||||
cityList.addAll(distList)
|
||||
// cityList << []
|
||||
}
|
||||
}
|
||||
|
||||
if(cityList){
|
||||
allList.addAll(cityList)
|
||||
println city +" "+cityList.size()
|
||||
}
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据")
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -35,7 +35,7 @@ class FjtResponseTimeAnaly extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//run();
|
||||
jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了?")
|
||||
jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了2?")
|
||||
}
|
||||
|
||||
|
||||
|
||||
+20
-8
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -82,20 +83,21 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
for(def dist : removeDists){
|
||||
def key = dist.city +"##"+dist.region+"##"+dist.name
|
||||
if(!distKeys.contains(key)){
|
||||
distKeys << key
|
||||
districts << dist
|
||||
}
|
||||
}
|
||||
// for(def dist : removeDists){
|
||||
// def key = dist.city +"##"+dist.region+"##"+dist.name
|
||||
// if(!distKeys.contains(key)){
|
||||
// distKeys << key
|
||||
// districts << dist
|
||||
// }
|
||||
// }
|
||||
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
|
||||
|
||||
def fzAddrMap = getFzAddress();
|
||||
|
||||
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
|
||||
saveOrUpdateFzDist(districts)
|
||||
// saveOrUpdateFzDist(districts)
|
||||
remarkRemoveDist(removeDists)
|
||||
def fzDistricts = getFzDistricts(city)
|
||||
|
||||
for(def rMap : fzDistricts){
|
||||
@@ -185,6 +187,15 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def remarkRemoveDist(removeDists){
|
||||
for(def rd : removeDists){
|
||||
def udate = new BasicDBObject()
|
||||
udate.put("status",1)
|
||||
udate.put("m_date", new Date())
|
||||
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
|
||||
}
|
||||
}
|
||||
|
||||
def getFzDistricts(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
@@ -311,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def s = str.toString();
|
||||
index++;
|
||||
if(s && s!="null"){
|
||||
s = s.replace('"','\\"')
|
||||
// if(s.toString().contains(",")){
|
||||
// s = csvHandlerStr(s.toString())
|
||||
//
|
||||
|
||||
+8
-6
@@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
if(weekendPriceObj.value_type!=1){
|
||||
if(dist.city == "¸£ÖÝ"){
|
||||
if(Math.abs(mom)*100>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
if(dist.region != "Âíβ"){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
}else{
|
||||
if(Math.abs(mom)*100>10){
|
||||
|
||||
+4
-1
@@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{
|
||||
// sb.append(s+ "@|@")
|
||||
// }
|
||||
// }
|
||||
sb.append(StringUtils.join(row.toArray(), "@|@"))
|
||||
def str = StringUtils.join(row.toArray(), "@|@")
|
||||
str = str.replace('"','\\"')
|
||||
sb.append(str)
|
||||
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
println index
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
@@ -11,26 +12,91 @@ import com.mongodb.DBCollection
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def datas = FileUtils.readLines("D:\\Êý¾Ý\\ÆäËü\\FJW_COMM_20221015.csv")
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def arrs = d.split("@")
|
||||
rets << ["commId":arrs[0], "avg_price":arrs[4]]
|
||||
}
|
||||
|
||||
for(def r : rets){
|
||||
if(r.avg_price){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("date", 202210)
|
||||
udata.put("avg", r.avg_price.toInteger())
|
||||
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
|
||||
}else{
|
||||
println r.commId +" ²»´æÔÚ"
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
|
||||
def city = RegionData.formatCity("余姚")
|
||||
d['D'] = city
|
||||
if(city){
|
||||
d['E'] = RegionData.formatRegion(city, d['C'])
|
||||
}
|
||||
|
||||
// if(!d['D']){
|
||||
// def citys = CommonDatautils.getCityByRegion(d['A'])
|
||||
// if(citys && citys.size()==1){
|
||||
// d['F'] = citys.get(0)
|
||||
// }
|
||||
// }
|
||||
})
|
||||
|
||||
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
|
||||
//def m = [:]
|
||||
//for(def d : datas){
|
||||
// m.put(d['B'], d)
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d->
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// if(dist){
|
||||
// def ret = m.get(dist._id.toString())
|
||||
// if(ret){
|
||||
// d['E'] = ret['J']
|
||||
// d['F'] = ret['N']
|
||||
// }
|
||||
|
||||
// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS)
|
||||
// if(dbo){
|
||||
// if(dbo.beike_esf_price){
|
||||
// d['G'] = dbo.beike_esf_price
|
||||
// }else if(dbo.beike_esf_price){
|
||||
// d['G'] = dbo.beike_esf_price
|
||||
// }
|
||||
//
|
||||
// if(dbo.maitian_esf_price){
|
||||
// d['H'] = dbo.maitian_esf_price
|
||||
// }
|
||||
// }
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd"))
|
||||
// if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
// d['L'] = dbo.value_type
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", 202304)
|
||||
return AVERAGE_STATISTICS.findOne(q)
|
||||
}
|
||||
//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
|
||||
//def rets = []
|
||||
//for(def d : datas){
|
||||
// def arrs = d.split("@")
|
||||
// rets << ["commId":arrs[0], "avg_price":arrs[4]]
|
||||
//}
|
||||
//
|
||||
//for(def r : rets){
|
||||
// if(r.avg_price){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("date", 202210)
|
||||
// udata.put("avg", r.avg_price.toInteger())
|
||||
// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
|
||||
// }else{
|
||||
// println r.commId +" 不存在"
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+5
-2
@@ -29,8 +29,11 @@ class AddAvgpriceBatch {
|
||||
}
|
||||
|
||||
def test(){
|
||||
//发重算均价命令
|
||||
addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd'))
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"])
|
||||
for(def dist : districts){
|
||||
//发重算均价命令
|
||||
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
|
||||
|
||||
+13
-2
@@ -96,17 +96,28 @@ class CityStatExport {
|
||||
}
|
||||
|
||||
def regionStat(){
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州']
|
||||
def ret = []
|
||||
ret << ["城市","区域","小区总量"]
|
||||
for(def city : citys){
|
||||
|
||||
def regions = CommonDatautils.getRegionListByCity(city)
|
||||
for(def r : regions){
|
||||
def row = []
|
||||
row << city
|
||||
row << r
|
||||
def districts = CommonDatautils.findDistricts(city, r, null)
|
||||
println city +" "+ r +" "+ districts.size()
|
||||
|
||||
row << CommonDatautils.queryCityRegionPrice(city, r, null)
|
||||
def obj = CommonDatautils.findRegionPriceRate(city, r)
|
||||
row << obj.price_coverage*100
|
||||
row << districts.size()
|
||||
|
||||
ret << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
|
||||
ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县")
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -272,6 +272,45 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processName(name){
|
||||
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
|
||||
if (name.contains('(') && name.contains(')')){
|
||||
def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.contains('(') && name.contains(')')){
|
||||
def rep_str = name.subsring(name.index('('),name.rfind(')')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('【')!=-1 && name.find('】')!=-1){
|
||||
def rep_str = name[name.index('【'):name.rfind('】')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('【')!=-1 && name.find('】')!=-1){
|
||||
def rep_str = name[name.index('【'):name.rfind('】')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('(')!=-1 && name.find(')')!=-1){
|
||||
def rep_str = name[name.index('('):name.rfind(')')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('「')!=-1 && name.find('」')!=-1){
|
||||
def rep_str = name[name.index('「'):name.rfind('」')+1]
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
static def processCarSite(car_site_count){
|
||||
def v = getValue(car_site_count.toString());
|
||||
if(v){
|
||||
@@ -579,6 +618,12 @@ class FiledStrand {
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}else if(estateType2=="写字楼"){
|
||||
if(estateCharege>=0.35 && estateCharege<=60){
|
||||
return estateCharege;
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
println "estateCharege ת²»ÁË "+estateCharege
|
||||
|
||||
@@ -567,7 +567,7 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
static def financialRating(city, address){
|
||||
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3"
|
||||
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
@@ -580,5 +580,79 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
|
||||
//源地址的完整性进行验证
|
||||
static def addrVerify(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// 省、市、区、街道、社区居委、POI补全
|
||||
static def complete(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
//五级(省/市/区/街道/社区居委)解析、补全
|
||||
static def division(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//识别地址中省、市、区、道路、POI同时进行地址验证
|
||||
static def structure(city, address){
|
||||
def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 招商银行测试
|
||||
* @param city
|
||||
* @param address
|
||||
* @return
|
||||
*/
|
||||
static def cmbTest(city, address){
|
||||
def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -69,29 +70,43 @@ import com.mongodb.DBObject
|
||||
//
|
||||
//
|
||||
//})
|
||||
println "开始统计"
|
||||
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
def prices = map.get(d['B'])
|
||||
if(!prices){
|
||||
prices = []
|
||||
map.put(d['B'], prices)
|
||||
}
|
||||
prices << d['G']
|
||||
}
|
||||
|
||||
def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
|
||||
def rows = []
|
||||
rows << ["区域","数量", "均价"]
|
||||
|
||||
regions.each{e->
|
||||
def prices = map.get(e)
|
||||
def price = (prices.sum()/prices.size()).toInteger()
|
||||
rows << [e, prices.size(), price]
|
||||
}
|
||||
//map.each{region,prices->
|
||||
// def price = (prices.sum()/prices.size()).toInteger()
|
||||
// rows << [region, prices.size(), price]
|
||||
//println "开始统计"
|
||||
//def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1)
|
||||
//def map = [:]
|
||||
//for(def d : datas){
|
||||
// def prices = map.get(d['B'])
|
||||
// if(!prices){
|
||||
// prices = []
|
||||
// map.put(d['B'], prices)
|
||||
// }
|
||||
// prices << d['G']
|
||||
//}
|
||||
ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果")
|
||||
//
|
||||
//def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
|
||||
//def rows = []
|
||||
//rows << ["区域","数量", "均价"]
|
||||
//
|
||||
//regions.each{e->
|
||||
// def prices = map.get(e)
|
||||
// def price = (prices.sum()/prices.size()).toInteger()
|
||||
// rows << [e, prices.size(), price]
|
||||
//}
|
||||
////map.each{region,prices->
|
||||
//// def price = (prices.sum()/prices.size()).toInteger()
|
||||
//// rows << [region, prices.size(), price]
|
||||
////}
|
||||
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果")
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\其它\\城市等级.xlsx",{d->
|
||||
// d['C'] = RegionData.formatCity(d['A'])
|
||||
//})
|
||||
|
||||
DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo);
|
||||
ExcelUtils.modify("D:\\数据\\其它\\城市等级.xlsx",{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("display", d['C'])
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("new_city_grade",d['E'].toString().toDouble().toInteger())
|
||||
SERVER_AREA_CONN.update(q, new BasicDBObject('$set', udata))
|
||||
})
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
|
||||
if(d['A'] && !d['E']){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
|
||||
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
|
||||
println d['A'] +" "+d['C']
|
||||
// def total = findDatas(d['A'], d['B'], d['C'])
|
||||
// d['D'] = total
|
||||
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
|
||||
d['E'] = c
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
def getAmapLocation(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name.contains(name) && d.type.contains("住宅")){
|
||||
total ++
|
||||
}
|
||||
})
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
def findDatas(def city, def region, def name){
|
||||
def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'"
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
total = d.count
|
||||
})
|
||||
return total;
|
||||
}
|
||||
|
||||
//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
|
||||
// def region = RegionData.formatCity(d['A'])
|
||||
// if(!region && d['A']){
|
||||
// println d['A']
|
||||
// }
|
||||
//})
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-04-01"
|
||||
static def statDate = "2023-05-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
@@ -76,24 +76,24 @@ MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S
|
||||
|
||||
|
||||
|
||||
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
hashSet << result
|
||||
}
|
||||
|
||||
|
||||
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
//def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
//def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
|
||||
//Calendar calendar = Calendar.getInstance()
|
||||
//calendar.setTime(startDate)
|
||||
//def count = 0
|
||||
//def hashSet = []
|
||||
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
//
|
||||
//OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
// def result = JSONObject.fromObject(doc.res_map.result)
|
||||
// def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
// data << item
|
||||
// hashSet << result
|
||||
//}
|
||||
//
|
||||
//
|
||||
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
//ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
|
||||
@@ -115,4 +115,27 @@ ExcelUtils.write("D:\\
|
||||
//
|
||||
//
|
||||
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
|
||||
|
||||
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230524", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]]
|
||||
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode,
|
||||
doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber]
|
||||
data << item
|
||||
hashSet << result
|
||||
}
|
||||
|
||||
|
||||
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
@@ -0,0 +1,207 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 按区域统计覆盖率
|
||||
*/
|
||||
|
||||
class XiamenStat {
|
||||
|
||||
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def DISTRICT_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
|
||||
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
def task = new XiamenStat();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d->
|
||||
if(d['D'] && d['D']=="全市"){
|
||||
def dbo = CommonDatautils.findCityInfo(d['C'].replace("市",""))
|
||||
if(dbo){
|
||||
def detail = dbo.district_detail
|
||||
if(detail){
|
||||
d['E'] = dbo.existing_district_count//数据量
|
||||
d['F'] = detail.township*100// 乡镇街道
|
||||
d['G'] = detail.address*100// 地址
|
||||
d['H'] = detail.addrAlias*100// 地址别名
|
||||
d['I'] = detail.name*100// 名称
|
||||
d['J'] = detail.alias*100// 别名
|
||||
d['K'] = detail.alias*100// 映射别名
|
||||
d['L'] = detail.lat2*100// 经纬度(百度)
|
||||
d['M'] = detail.lat*100// 经纬度(高德)
|
||||
d['N'] = detail.estateType2*100// 物业类型
|
||||
d['O'] = detail.buildType*100// 建筑类型
|
||||
d['P'] = detail.streetCommunity*100// 街道社区
|
||||
d['Q'] = detail.committee*100// 居委会
|
||||
d['R'] = detail.businessQuarter*100// 商圈
|
||||
d['S'] = detail.schools*100// 对口小学
|
||||
d['T'] = detail.kindergartens*100// 对口幼儿园
|
||||
d['U'] = detail.loudongCount*100// 楼栋总数
|
||||
d['V'] = detail.householdCount*100// 户数
|
||||
d['W'] = detail.estateCharge*100// 物业费
|
||||
d['X'] = detail.carSiteCount*100// 停车位
|
||||
d['Y'] = ""// 电梯
|
||||
d['Z'] = detail.propertyLife*100// 产权年限
|
||||
d['AA'] = detail.houseProperty*100// 产权性质
|
||||
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
|
||||
d['AC'] = detail.landProperty*100// 小区性质
|
||||
d['AD'] = detail.finishBuildingDate*100// 竣工时间
|
||||
d['AE'] = detail.openDate*100// 交房时间
|
||||
d['AF'] = detail.allArea*100// 占地面积
|
||||
d['AG'] = detail.buildArea*100// 建筑面积
|
||||
d['AH'] = detail.cubagePercent*100// 容积率
|
||||
d['AI'] = detail.virescencePercent*100// 绿化率
|
||||
d['AJ'] = detail.mainType*100// 主力户型
|
||||
d['AK'] = detail.mainTypeArea*100// 主力户型面积
|
||||
d['AL'] = detail.developer*100// 开发商
|
||||
d['AM'] = detail.estateCompany*100// 物业公司
|
||||
d['AN'] = dbo.price_coverage*100// 小区当前均价
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def fields = DISTRICT_FIELDS_MAP.keySet().asList()
|
||||
def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields)
|
||||
def detail = districtDetail(regionDistricts.size(), regionDistricts);
|
||||
if(detail){
|
||||
d['E'] = regionDistricts.size()//数据量
|
||||
d['F'] = detail.township*100// 乡镇街道
|
||||
d['G'] = detail.address*100// 地址
|
||||
d['H'] = detail.addrAlias*100// 地址别名
|
||||
d['I'] = detail.name*100// 名称
|
||||
d['J'] = detail.alias*100// 别名
|
||||
d['K'] = detail.alias*100// 映射别名
|
||||
d['L'] = detail.lat2*100// 经纬度(百度)
|
||||
d['M'] = detail.lat*100// 经纬度(高德)
|
||||
d['N'] = detail.estateType2*100// 物业类型
|
||||
d['O'] = detail.buildType*100// 建筑类型
|
||||
d['P'] = detail.streetCommunity*100// 街道社区
|
||||
d['Q'] = detail.committee*100// 居委会
|
||||
d['R'] = detail.businessQuarter*100// 商圈
|
||||
d['S'] = detail.schools*100// 对口小学
|
||||
d['T'] = detail.kindergartens*100// 对口幼儿园
|
||||
d['U'] = detail.loudongCount*100// 楼栋总数
|
||||
d['V'] = detail.householdCount*100// 户数
|
||||
d['W'] = detail.estateCharge*100// 物业费
|
||||
d['X'] = detail.carSiteCount*100// 停车位
|
||||
d['Y'] = ""// 电梯
|
||||
d['Z'] = detail.propertyLife*100// 产权年限
|
||||
d['AA'] = detail.houseProperty*100// 产权性质
|
||||
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
|
||||
d['AC'] = detail.landProperty*100 // 小区性质
|
||||
d['AD'] = detail.finishBuildingDate*100// 竣工时间
|
||||
d['AE'] = detail.openDate*100// 交房时间
|
||||
d['AF'] = detail.allArea*100// 占地面积
|
||||
d['AG'] = detail.buildArea*100// 建筑面积
|
||||
d['AH'] = detail.cubagePercent*100// 容积率
|
||||
d['AI'] = detail.virescencePercent*100// 绿化率
|
||||
d['AJ'] = detail.mainType*100// 主力户型
|
||||
d['AK'] = detail.mainTypeArea*100// 主力户型面积
|
||||
d['AL'] = detail.developer*100// 开发商
|
||||
d['AM'] = detail.estateCompany*100// 物业公司
|
||||
// d['AN'] = ""// 小区当前均价
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def districtDetail(def disCount, def regionDistricts) {
|
||||
|
||||
def countMap = [:]
|
||||
for(def dbo : regionDistricts) {
|
||||
DISTRICT_FIELDS_MAP.each() {key,skey->
|
||||
if (isValid(dbo.get(key))) {
|
||||
if(countMap.containsKey(key)) {
|
||||
countMap.put(key, countMap.get(key) + 1)
|
||||
}else {
|
||||
countMap.put(key, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def districtDetail = [:];
|
||||
DISTRICT_FIELDS_MAP.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
districtDetail.put(skey, roundDouble(countMap.get(key), disCount))
|
||||
}else {
|
||||
districtDetail.put(skey, 0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(countMap.containsKey("lat") && countMap.containsKey("lng")) {
|
||||
districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount));
|
||||
}else {
|
||||
districtDetail.put('latLng', 0);
|
||||
}
|
||||
|
||||
if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) {
|
||||
districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount));
|
||||
}else {
|
||||
districtDetail.put('latLng2', 0);
|
||||
}
|
||||
return districtDetail;
|
||||
}
|
||||
|
||||
|
||||
def CONTAINS_KWS = ["...", "暂无", "待定", "未知"]
|
||||
def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."]
|
||||
def isValid(def value) {
|
||||
if (!value) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (value instanceof String) {
|
||||
value = value.trim()
|
||||
for(String ck : CONTAINS_KWS) {
|
||||
if(value.contains(ck)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for(String ek : EQUALS_KWS) {
|
||||
if(ek.equals(value)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Number) {
|
||||
if (value <= 0) {
|
||||
return false;
|
||||
}
|
||||
} else if (value instanceof Collection || value instanceof Map) {
|
||||
if (value.size() == 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
double roundDouble(int a, int base) {
|
||||
if(a == 0 || base == 0) {
|
||||
return 0
|
||||
}
|
||||
return ((double)a / (double)base * 10000.0).toInteger() / 10000.0
|
||||
}
|
||||
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 通过房源更改电梯
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class AddDistrictElevatorBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictElevatorBatch(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def dists = CommonDatautils.findAllDistricts(c, "district", ["city","region","name","alias", "build_type", "estate_type2", "is_elevator"])
|
||||
addEvevatorProcess(c, dists)
|
||||
}
|
||||
}
|
||||
|
||||
def addEvevatorProcess(c, dists){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
println "小区量:"+dists.size()
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(!dist.is_elevator){
|
||||
def houses = CommonDatautils.findHouses(dist.city, dist.region, null, 20200101, 20230701, dist.name, dist.alias, dist.estate_type2)
|
||||
def haveHouses = []
|
||||
for(def h : houses){
|
||||
if(h.height){
|
||||
haveHouses << h
|
||||
}
|
||||
}
|
||||
def newHouses = analyElevator(haveHouses, dist.estate_type2)
|
||||
if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){
|
||||
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size()
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
if(dist.estate_type2 == "别墅"){
|
||||
udata.put("is_elevator", false)
|
||||
}else{
|
||||
udata.put("is_elevator", true)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
rets << dist
|
||||
}else{
|
||||
if(dist.build_type && dist.build_type.contains("高层")){
|
||||
BasicDBObject udata = new BasicDBObject('is_elevator',true)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
rets << dist
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
println c +" "+ rets.size()
|
||||
// return retMap
|
||||
}
|
||||
|
||||
def analyElevator(houses, estateType){
|
||||
def heights = []
|
||||
for(def h : houses){
|
||||
if(estateType=="别墅"){
|
||||
if(h.height && h.height<=4) {
|
||||
heights << h.height
|
||||
}
|
||||
}else{
|
||||
if(h.height && h.height>=10){
|
||||
heights << h.height
|
||||
}
|
||||
}
|
||||
}
|
||||
return heights
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+2
-2
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 小区字段入库(自动入库)
|
||||
* 小区字段填充入库(自动入库)
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictFieldAddBatch extends BaseBatch{
|
||||
@@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = ["漳州"]//CommonDatautils.findAllCity().reverse()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
|
||||
def analysRets = []
|
||||
|
||||
|
||||
+1
-1
@@ -95,7 +95,7 @@ class FangjiaClientUtils {
|
||||
try {
|
||||
if(address){
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 写字楼字段清洗入库(自动入库)
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static main(args) {
|
||||
runBatch(new OfficeFieldCleanBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
for(def c : citys){
|
||||
def offices = findAllOffice(c)
|
||||
strandField(offices)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def strandField(offices){
|
||||
def fields = ["property_fee","build_area","","","","","","","","","",""]
|
||||
for(def office : offices){
|
||||
fields.each{
|
||||
def newVal = null
|
||||
if(it == "property_fee"){
|
||||
newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼")
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
|
||||
if(it =="all_area" || it =="build_area"){
|
||||
newVal = FiledStrand.processArea(office.get(it))
|
||||
if(newVal && newVal > 5000){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
|
||||
if(it =="name"){
|
||||
newVal = FiledStrand.processName(office.get(it))
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findAllOffice(city){
|
||||
def sql = "select * from office where city='${city}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+43
-1
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -15,7 +16,48 @@ class SameDistName {
|
||||
|
||||
static main(args) {
|
||||
def task = new SameDistName()
|
||||
task.run()
|
||||
// task.run()
|
||||
//地址和地址别名完全一致
|
||||
// task.addressPocess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addressPocess(){
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
for(def c : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"])
|
||||
if(dists){
|
||||
def index = 0
|
||||
for(def dist : dists){
|
||||
if(dist.address){
|
||||
// if(dist.address.toString().endsWith("弄")){
|
||||
// dist.address = dist.address.toString().replace("弄","号")
|
||||
// }
|
||||
if(dist.addr_alias && dist.addr_alias.contains(dist.address)){
|
||||
def new_address_alias = []
|
||||
for(def addr : dist.addr_alias){
|
||||
if(!new_address_alias.contains(addr) && addr!=dist.address){
|
||||
new_address_alias << addr
|
||||
}
|
||||
}
|
||||
index ++
|
||||
|
||||
println "[${index}] "+dist.city +" "+dist.name+" 旧:"+dist.addr_alias+" 新:"+ new_address_alias
|
||||
|
||||
def d = new BasicDBObject()
|
||||
d.put("_id", dist._id)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("addr_alias", new_address_alias)
|
||||
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+3
@@ -144,5 +144,8 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
|
||||
|
||||
public static void main(def agrs){
|
||||
doMatch("上海", "静安", "金家巷西弄", null)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+12
-5
@@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&
|
||||
stat()
|
||||
|
||||
def stat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1)
|
||||
def map=[:]
|
||||
for(def d : datas){
|
||||
def count = map.get(d['C'])
|
||||
@@ -69,8 +69,8 @@ def findLogs(){
|
||||
|
||||
def findFangjiaLogs(){
|
||||
|
||||
def startDate = DateUtil.parseDate("20230329", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230429", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230531", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def list = []
|
||||
@@ -84,14 +84,21 @@ def findFangjiaLogs(){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
|
||||
// def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
|
||||
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
|
||||
|
||||
def q = QueryBuilders.fuzzyQuery("request", "/enquire")
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
|
||||
println houses.size()
|
||||
|
||||
if (houses) {
|
||||
for (def d : houses) {
|
||||
def reqStr = d.request.toString()
|
||||
if(!reqStr.contains("/enquire")){
|
||||
continue
|
||||
}
|
||||
def mid = null
|
||||
if (reqStr.contains("mid=")) {
|
||||
|
||||
@@ -107,5 +114,5 @@ def findFangjiaLogs(){
|
||||
startDate = nextDate
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志")
|
||||
ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志")
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class HouseAnaly {
|
||||
|
||||
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new HouseAnaly()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allRets = []
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"])
|
||||
def rets = processing(districts)
|
||||
println "[${city}] 写文件 "+rets.size()
|
||||
allRets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析")
|
||||
}
|
||||
|
||||
def processing(dists){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"]
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(dist.alias && dist.alias.size()>=1){
|
||||
def ret = analy(dist)
|
||||
if(ret){
|
||||
rets.addAll(ret)
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def analy(dist){
|
||||
def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是");
|
||||
def alias = dist.alias
|
||||
for(def aliasName : alias){
|
||||
if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){
|
||||
def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否");
|
||||
if(aliaRet){
|
||||
masterRet.addAll(aliaRet)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(masterRet.size()>=2){
|
||||
def masterAvgPrice = 0
|
||||
def masterMidPrice = 0
|
||||
def back = false
|
||||
for(def row : masterRet){
|
||||
if(row.get(4)=="是"){
|
||||
masterAvgPrice = row.get(8)
|
||||
masterMidPrice = row.get(9)
|
||||
}else{
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
if(masterAvgPrice>0){
|
||||
pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice)
|
||||
}
|
||||
if(masterMidPrice>0){
|
||||
pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice)
|
||||
}
|
||||
if(pre1>=20 || pre2>=20){
|
||||
back = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if(back){
|
||||
return masterRet
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量
|
||||
def runAnalying(id, city, region, name, flag){
|
||||
def houses = findHouse(city, region, name);
|
||||
def rets = []
|
||||
if(houses && houses.size()>=5){
|
||||
def prices = []
|
||||
for(def house : houses){
|
||||
prices << house.avg_price
|
||||
}
|
||||
prices = prices.sort()
|
||||
def average = (int)(prices.sum() / prices.size())
|
||||
def midPrice = getMidPrice(prices)
|
||||
def price10 = null
|
||||
def price90 = null
|
||||
if(houses.size()>=10){
|
||||
def postion = Math.round(houses.size()/10)
|
||||
price10 = prices.get((int)(postion-1))
|
||||
price90 = prices.get((int)(houses.size()-postion-1))
|
||||
}
|
||||
def countMap = classType(houses)
|
||||
|
||||
// def pre = java.lang.Math.abs((midPrice-average)*100/average)
|
||||
// if(pre>=20){
|
||||
rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")]
|
||||
// }
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def classType(houses){
|
||||
def district = 0
|
||||
def villa = 0
|
||||
def business = 0
|
||||
for(def h : houses){
|
||||
if(h.estate_type){
|
||||
if(h.estate_type == "别墅"){
|
||||
villa = villa + 1
|
||||
continue
|
||||
}else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){
|
||||
business = business + 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
def valla = isVilla(h.area, h.name, h.height, h.room)
|
||||
if(valla){
|
||||
villa = villa + 1
|
||||
continue
|
||||
}
|
||||
district = district + 1
|
||||
}
|
||||
return ["district":district, "villa":villa, "business":business]
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public boolean isVilla(area, name, height, room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
def getMidPrice(def prices){
|
||||
def mid = prices.size()/2
|
||||
if(mid > (int)mid){
|
||||
return prices.get((int)mid)
|
||||
}else{
|
||||
def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1))
|
||||
return (int)(total/2)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def findHouse(def city, def region, def name){
|
||||
def houses = []
|
||||
try{
|
||||
name = escapeExprSpecialWord(name)
|
||||
String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'"
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type]
|
||||
houses << r
|
||||
// println r
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return houses
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
|
||||
}
|
||||
+13
-3
@@ -21,13 +21,23 @@ class GaoDeMapSearchTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new GaoDeMapSearchTest();
|
||||
def path = "D:\\数据\\光大\\光大测试-V2.xlsx"
|
||||
def path = "D:\\数据\\富民\\富民测试500.xlsx"
|
||||
|
||||
// task.structureV2(path)
|
||||
// task.match(path)
|
||||
// task.mapSearch(path)
|
||||
task.baiduMapSearch(path)
|
||||
task.checkTrue(path)
|
||||
// task.baiduMapSearch(path)
|
||||
// task.checkTrue(path)
|
||||
task.addFinancialLevel(path)
|
||||
}
|
||||
|
||||
def addFinancialLevel(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A'])
|
||||
if(results){
|
||||
row['V'] = results.consumptionRating
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def checkTrue(path){
|
||||
|
||||
+73
-27
@@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome {
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
|
||||
//广州,佛山 重庆
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
@@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome {
|
||||
println city +" 已运行"
|
||||
continue
|
||||
}
|
||||
def newCity = RegionData.formatCity(city)
|
||||
if(!newCity){
|
||||
if(city=="昆山"){
|
||||
newCity = "苏州"
|
||||
}
|
||||
println city +" 无法格式化"
|
||||
}
|
||||
def url = m[1]
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
@@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome {
|
||||
def href = url+ r.attr("href")
|
||||
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, city, href, regionText)
|
||||
if(city == "昆山"){
|
||||
regionText = "昆山"
|
||||
}
|
||||
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
@@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, region)
|
||||
def dbo = findTrade(durl , "贝壳找房")
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, region){
|
||||
def parse(city,e, region, driver){
|
||||
try {
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
if(title.contains("车位")){
|
||||
detail.type = "车位"
|
||||
}
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
|
||||
@@ -266,28 +289,48 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
detail.url = detailUrl
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
|
||||
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构",""))
|
||||
detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡",""))
|
||||
|
||||
detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","")
|
||||
detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属",""))
|
||||
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
|
||||
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限",""))
|
||||
if(detail.total_price && detail.avg_price){
|
||||
|
||||
detail.url = detailUrl
|
||||
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
|
||||
detail.floor_info = floorStr
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorInfo && floorInfo.isNumber()) {
|
||||
detail.height = floorInfo.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -392,16 +435,16 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def findTrade(def url, source){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city", detail.city)
|
||||
// q.put("region", detail.region)
|
||||
// q.put("district_name", detail.district_name)
|
||||
// q.put("area", detail.area)
|
||||
// q.put("total_price", detail.total_price)
|
||||
q.put("url", detail.url)
|
||||
q.put("url", url)
|
||||
// q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
q.put("source", source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
@@ -448,10 +491,13 @@ public class BeiKeCrawlByChrome {
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
Thread.sleep(15000)
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
class DataFormatUtils {
|
||||
|
||||
static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000]
|
||||
|
||||
static def parseLift(def str){
|
||||
if(str && str.contains("梯")){
|
||||
def lift = str.toString().substring(0, str.toString().indexOf("梯"))
|
||||
if(lift){
|
||||
def val = chineseIntegerPart2Digit(lift)
|
||||
if(val){
|
||||
return (int)val
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def parseHousehold(def str){
|
||||
if(str && str.contains("梯") && str.contains("户")){
|
||||
def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户"))
|
||||
def val = chineseIntegerPart2Digit(household)
|
||||
if(val){
|
||||
return (int)val
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def chineseIntegerPart2Digit(String chineseDigit)
|
||||
{
|
||||
Double finalDigit = 0.0;
|
||||
Integer r = 1;
|
||||
|
||||
for(int i = chineseDigit.length()-1; i>=0;i--)
|
||||
{
|
||||
Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i)));
|
||||
|
||||
if( val >= 10 && i == 0 )
|
||||
{
|
||||
if (val > r)
|
||||
{
|
||||
r = val;
|
||||
finalDigit = finalDigit +val;
|
||||
}
|
||||
else
|
||||
{
|
||||
r = r * val;
|
||||
}
|
||||
}
|
||||
else if(val >= 10)
|
||||
{
|
||||
if (val > r)
|
||||
{
|
||||
r = val;
|
||||
}
|
||||
else
|
||||
{
|
||||
r = r * val;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
finalDigit = finalDigit + r*val;
|
||||
}
|
||||
}
|
||||
return finalDigit;
|
||||
}
|
||||
|
||||
static String getValue(String s) {
|
||||
if (s && s.contains("暂无")) {
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
+129
-72
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
@@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -196,9 +197,22 @@ public class LianjiaCrawlByChrome {
|
||||
println city +" 已运行"
|
||||
continue
|
||||
}
|
||||
def newCity = RegionData.formatCity(city)
|
||||
if(!newCity){
|
||||
if(city=="昆山"){
|
||||
newCity = "苏州"
|
||||
}
|
||||
println city +" 无法格式化"
|
||||
}
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
driver.get(newUrl)
|
||||
try{
|
||||
driver.get(newUrl)
|
||||
}catch(Exception e){
|
||||
println newUrl +" 链接有问题!"
|
||||
continue
|
||||
}
|
||||
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
@@ -208,9 +222,11 @@ public class LianjiaCrawlByChrome {
|
||||
for(def r : regions){
|
||||
def regionText = r.text()
|
||||
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
|
||||
|
||||
if(city == "昆山"){
|
||||
regionText = "昆山"
|
||||
}
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, city, href, regionText)
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
@@ -282,13 +298,21 @@ public class LianjiaCrawlByChrome {
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, regionText)
|
||||
if(dealDate){
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
println url +" 成交时间有误!"
|
||||
}
|
||||
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
@@ -302,85 +326,116 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, regionText){
|
||||
def parse(city,e, regionText, driver){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
if(regionText){
|
||||
detail.region = regionText
|
||||
}else{
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
}
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
def room = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
if(room && room.isNumber()){
|
||||
detail.room = room.toInteger()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
if(title.contains("车位")){
|
||||
detail.type = "车位"
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
if(hall && hall.isNumber()){
|
||||
detail.hall = hall.toInteger()
|
||||
}
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
if(regionText){
|
||||
detail.region = regionText
|
||||
}else{
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
}
|
||||
}
|
||||
|
||||
if("车位"!=roomStr){
|
||||
if(arrs && arrs.size()>=3){
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
if(!detail.region){
|
||||
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
|
||||
str = str.substring(0, str.indexOf("',"))
|
||||
def locArrs = str.split(",")
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
|
||||
if(gmap){
|
||||
detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat)
|
||||
}
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
def room = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
if(room && room.isNumber()){
|
||||
detail.room = room.toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
if(hall && hall.isNumber()){
|
||||
detail.hall = hall.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
if("车位"!=roomStr){
|
||||
if(arrs && arrs.size()>=3){
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
}
|
||||
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = detailUrl
|
||||
|
||||
def detailDiv = JQueryUtils.select(".introContent", detailDoc)
|
||||
|
||||
def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层",""))
|
||||
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
|
||||
detail.floor_info = floorStr
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr2 && floorStr2.isNumber()) {
|
||||
detail.height = floorStr2.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -484,16 +539,16 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def findTrade(def url, source){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city", detail.city)
|
||||
// q.put("region", detail.region)
|
||||
// q.put("district_name", detail.district_name)
|
||||
// q.put("area", detail.area)
|
||||
// q.put("total_price", detail.total_price)
|
||||
q.put("url", url)
|
||||
// q.put("trade_date", detail.trade_date)
|
||||
q.put("url", detail.url)
|
||||
q.put("source", detail.source)
|
||||
q.put("source", source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
@@ -530,12 +585,14 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
Thread.sleep(15000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
}
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.fangjia.script.gj.region
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 格式化其它网站区域
|
||||
*/
|
||||
|
||||
|
||||
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
|
||||
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl)
|
||||
|
||||
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1)
|
||||
for(def d : datas){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("recognizeCity", d['F'])
|
||||
q.put("recognizeRegion", d['A'])
|
||||
def count = SERVER_AREA_COLL.count(q)
|
||||
if(!count){
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("city", d['A'])
|
||||
obj.put("recognizeRegion", d['A'])
|
||||
obj.put("recognizeCity", d['F'])
|
||||
obj.put("createdDate", new Date())
|
||||
SERVER_AREA_COLL.save(obj)
|
||||
}else{
|
||||
println d['A']+" "+ d['F'] +" 已存在"
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -58,8 +58,9 @@ class CZDataBatch extends BaseBatch{
|
||||
def commids = findAllJnBankDatas(city)
|
||||
|
||||
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
|
||||
districtRun(districts, statMap, commids);
|
||||
FzDataUtils.removeNoUpdate(city, districts)
|
||||
districtRun(districts, statMap, commids);
|
||||
|
||||
|
||||
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
|
||||
shops = mergeShop(shops);
|
||||
|
||||
+12
-3
@@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
|
||||
|
||||
if(dists){
|
||||
for(def d : dists){
|
||||
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
|
||||
def distObj = null
|
||||
if(d.comm_id){
|
||||
distObj = CommonDatautils.findDistrictById(d.comm_id)
|
||||
}else{
|
||||
distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
|
||||
}
|
||||
if(distObj && distObj._id){
|
||||
FzDataUtils.converDistrict(distObj, null, null);
|
||||
if(d.type && d.type==1){
|
||||
FzDataUtils.removeById(d.comm_id)
|
||||
}else{
|
||||
FzDataUtils.converDistrict(distObj, null, null);
|
||||
}
|
||||
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
|
||||
updateTaskStatus(d.id, 2);
|
||||
}
|
||||
@@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("status", 5);//未处理
|
||||
TASK_COLL.find(q).each {d->
|
||||
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
|
||||
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
+7
@@ -119,6 +119,13 @@ class FzDataUtils {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static def removeById(id){
|
||||
def del_data = new BasicDBObject()
|
||||
del_data.put("num_id", id)
|
||||
DIST_COLL.remove(del_data)
|
||||
}
|
||||
|
||||
static def saveDistrict(def m){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("num_id",m.num_id)
|
||||
|
||||
+7
-4
@@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "역迦속潼±${city}×"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// if(city == "¸£ÖÝ"){
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!dbo){
|
||||
if(!dbo && isStandardRoad(copyObj.address)){
|
||||
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||
if(result.size()==1){
|
||||
dbo = result[0];
|
||||
@@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
// qdata.put("city","¸£ÖÝ")
|
||||
qdata.put("city","¸£ÖÝ")
|
||||
// qdata.put("source","갛앙와")
|
||||
// qdata.put("name",Pattern.compile("묠소酪.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
}
|
||||
@Override
|
||||
void process(Object args) {
|
||||
println SHORT_DATE
|
||||
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
|
||||
@@ -237,7 +237,7 @@ public class DbUtils{
|
||||
def lng = district.lng / 10000000000.0
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
|
||||
sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0));
|
||||
query.append('$sd', sdq)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
|
||||
Reference in New Issue
Block a user