Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
zhaojixiang@fangjia.com
2023-07-06 13:45:49 +08:00
55 changed files with 2668 additions and 296 deletions
@@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ServerAreaMap import com.fangjia.script.util.ServerAreaMap
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/** /**
* Created by dxy on 2016/12/15. * Created by dxy on 2016/12/15.
*/ */
@@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch {
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def DISTRICT_STAT_MAP = [:] def DISTRICT_STAT_MAP = [:]
public void process(Object args) { public void process(Object args) {
def cities = ServerAreaMap.findAllCities()
def cities = ["深圳"]//ServerAreaMap.findAllCities()
for(String city : cities) { for(String city : cities) {
DISTRICT_STAT_MAP.clear() // DISTRICT_STAT_MAP.clear()
Map<String, List<String>> regions = RegionData.getAllRegions(city) // Map<String, List<String>> regions = RegionData.getAllRegions(city)
regions.each {region, alias-> // regions.each {region, alias->
processRegionRank(city, region) // processRegionRank(city, region)
} // }
processCityRank(city) // processCityRank(city)
processCityRentRank(city)
} }
} }
@@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch {
return ret return ret
} }
def processCityRentRank(city){
def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd")
def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth)
def prices = []
distRentsMap.each{key, price->
if(!prices.contains(price)){
prices << price
}
}
// prices = prices.unique()
prices = prices.sort().reverse()
def totalCount = new AtomicLong()
def keys = distRentsMap.keySet().asList()
final CountDownLatch latch = new CountDownLatch(keys.size())
// for(def i=0;i<keys.size();i++){
// final def key = keys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
// totalCount.addAndGet(1)
// upateDistrict(key, distRentsMap, prices)
// println totalCount.get()+" "+ key
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// })
// }
// latch.await()
def index = 0
distRentsMap.each{key, price->
index ++
def arrs = key.split("##");
double percentile = calculatePercentile(prices, price);
DBObject q = new BasicDBObject("city", arrs[0])
q.append("region", arrs[1])
q.append("district_name", arrs[2])
DBObject udata = new BasicDBObject()
udata.put("rent_city_rank", (int)percentile);
udata.put("m_date", new Date())
DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
println index +" "+ arrs[2]
}
}
def upateDistrict(key, distRentsMap, prices){
def arrs = key.split("##");
def price = distRentsMap.get(key)
double percentile = calculatePercentile(prices, price);
DBObject q = new BasicDBObject("city", arrs[0])
q.append("region", arrs[1])
q.append("district_name", arrs[2])
DBObject udata = new BasicDBObject()
udata.put("rent_city_rank", (int)percentile);
udata.put("m_date", new Date())
// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
public static int calculatePercentile(def values, def targetValue) {
// 找出数字在List中的排名
def rank = values.indexOf(targetValue) + 1
// 计算百分位数
def percentile = (int)((rank / values.size()) * 100)
return percentile;
}
static main(args) { static main(args) {
runBatch(new DistrictStatRankBatch(), args) runBatch(new DistrictStatRankBatch(), args)
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/** /**
* 深圳公众号成交数据录入 * 深圳公众号成交数据录入
*/ */
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1) def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -51,7 +51,19 @@ for(def d : datas){
map.region = d['B'] map.region = d['B']
def roomStr = d['E'] def roomStr = d['E']
if(roomStr){ if(roomStr){
map.room = roomStr.toString().toDouble().toInteger() if(roomStr.contains("-")){
def arrs = roomStr.split("-")
try{
map.room = arrs[0].toString().toInteger()
}catch(Exception e){
println "room error"
}
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
}else{
map.room = roomStr.toString().toDouble().toInteger()
}
} }
map.area = d['F'].toString().toDouble() map.area = d['F'].toString().toDouble()
@@ -59,7 +71,13 @@ for(def d : datas){
map.avg_price = (int)(d['H'].toString().toDouble()) map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000) map.sell_price = (int)(d['I'].toString().toDouble()*10000)
def trade_date = d['J'] def trade_date = d['J']
map.trade_date = trade_date if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
map.source = "深房家族" map.source = "深房家族"
map.create_date = new Date() map.create_date = new Date()
map.m_date = new Date() map.m_date = new Date()
@@ -77,7 +77,6 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle" println "txt 生成->导Oracle"
delPgData() delPgData()
saveToPgSql(rows) saveToPgSql(rows)
@@ -66,7 +66,8 @@ class BigdataDealBatch extends BaseBatch{
curDate = args[1].toString().toInteger() curDate = args[1].toString().toInteger()
} }
println "curDate:"+curDate println "curDate:"+curDate
// for(def city : citys){ // def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
//// if(!ccbCitys.contains(city)){ //// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city); // addDealPriceMatch(city);
//// } //// }
@@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
final CountDownLatch latch = new CountDownLatch(trades.size()) final CountDownLatch latch = new CountDownLatch(trades.size())
def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger count_ = new AtomicInteger(0)
@@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address); rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
} }
} }
if(rMap){ if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
} }
} }
@@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter);
//测试通过匹配接口配房源效果 //测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
} }
@@ -428,7 +428,7 @@ class CcbCommonUtils {
public static def findAllDistrictByCity(def city, def cat, def fields){ public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put("visible",0) q.put("status",0)
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
@@ -28,9 +28,11 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess() def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice(); // task.addLianJiaPrice();
task.deleteThirdPartyData() // task.deleteThirdPartyData()//删除第三方
// task.delDealByPe() // task.delDealByPe()
// task.delDealData() // task.delDealData()
task.exportThirdParty()
} }
def delDealByPe(){ def delDealByPe(){
@@ -372,15 +374,29 @@ class DealPriceDataProcess {
return map return map
} }
def exportThirdParty(){
def citys = CcbCommonUtils.citys_all
def results = []
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
for(def city : citys) {
def rets = findPeGteThirdParty3(city, 202305)
for(def map : rets){
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
}
println city +" "+ rets.size()
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方")
}
def deleteThirdPartyData(){ def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all def citys = CcbCommonUtils.citys_all
def month = 202303 def month = 202304
def rows = [] def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet() def set = new HashSet()
for(def city : citys) { for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202304) def rets = findPeGteThirdParty2(city, 202305)
def total = 0 def total = 0
println city +" "+rets.size() println city +" "+rets.size()
if(rets){ if(rets){
@@ -431,6 +447,28 @@ class DealPriceDataProcess {
return rets return rets
} }
//城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE
def findPeGteThirdParty3(cityName, yearmonth){
def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15"
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
map.cityid = r.urbn_id
map.city = r.urbn_nm
map.data_id = r.gujia_fsid
map.region = r.gujia_rgon_nm
map.comm_id = r.gujia_comm_id
map.district_name = r.gujia_comm_nm
map.type = r.gujia_bds_clc_tp
map.clc_price = r.gujia_bds_avprc
map.price = r.gujiabds_to_mkprc_mny
map.ppe = r.gujia_mns_rto
rets << map
});
return rets
}
def delDealData(){ def delDealData(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d -> ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d ->
@@ -79,9 +79,9 @@ class ExportOracleAllDataTest {
// exportCommCaseDetailGj()// // exportCommCaseDetailGj()//
// // ok文件 // // ok文件
exportInOk() // exportInOk()
SSHHelperTest jschCommand = new SSHHelperTest(); // SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); // System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
} }
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice() def task = new FjwPricePkCcbPrice()
// task.run() // task.run()
// task.updatePrice() // task.updatePrice()
// task.addFjwPrice() task.addFjwPrice()
// task.addCcbPrice()//oracle // task.addCcbPrice()//oracle
// task.addCcbPriceFromGuided() // task.addCcbPriceFromGuided()
@@ -42,7 +42,7 @@ class FjwPricePkCcbPrice {
// task.exportFjwPriceAndCcbPrice() // task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice() // task.addNewDistPrice()
task.addCcbPrice2()//按id查 // task.addCcbPrice2()//按id查
// task.addCcbPrice3()//按city,region,name查 // task.addCcbPrice3()//按city,region,name查
// task.standRegion() // task.standRegion()
// task.addGuidPrice() // task.addGuidPrice()
@@ -217,17 +217,19 @@ class FjwPricePkCcbPrice {
} }
def addFjwPrice(){ def addFjwPrice(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d->
if(!d['K']){ if(!d['N']){
def dist = CommonDatautils.findDistrictById(d['C']) def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){ if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd")) d['N'] = dist.region
if(dbo){ // def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd"))
d['K'] = dbo.avg_price // if(dbo){
} // d['K'] = dbo.avg_price
// }
} }
} }
}) })
} }
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23") def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
@@ -83,8 +83,8 @@ class GuiDingPriceBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
saveToPgSql(rows) saveToPgSql(rows)
//
//更新静态数据表 // //更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
} }
@@ -33,8 +33,8 @@ import java.util.regex.*;
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", //def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
@@ -1287,18 +1287,18 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d-> ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){ if(d['A']){
//// def region = RegionData.formatRegion(d['A'], d['B']) def region = RegionData.formatRegion(d['B'], d['C'])
//// if(region){ if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//// if(dist){ if(dist){
//// d['E'] = dist.region d['G'] = dist.region
//// d['F'] = dist._id.toString() d['H'] = dist._id.toString()
//// } }
//// } }
// if(!d['F']){ // if(!d['F']){
// def dist = CommonDatautils.findDistrictById(d['A']) // def dist = CommonDatautils.findDistrictById(d['A'])
//// ////
@@ -1308,30 +1308,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
// d['F'] = dist.status // d['F'] = dist.status
// } // }
// } // }
//
// }
//},0)
def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
if(count && count==1){
def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
d['E'] = dist.city
} }
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['J'] = dist.status
// }
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// d['F'] = newRegion
},0) },0)
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
// d['E'] = dist.city
// }
//
//// def dist = CommonDatautils.findDistrictById(d['A'])
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['J'] = dist.status
//// }
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
//},0)
//ExcelUtils.modify(fileName, {d-> //ExcelUtils.modify(fileName, {d->
// if(d['A']){ // if(d['A']){
// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" // def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'"
@@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
} }
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
saveToPgSql(rows) saveToPgSql(rows)
@@ -45,6 +45,7 @@ class TradeStatLog {
// task.thirdPartyStat() // task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR) // println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat() // task.averageStat()
// task.deleteDistrict() // task.deleteDistrict()
// task.deleteDealData() // task.deleteDealData()
// task.deleteDealData2() // task.deleteDealData2()
@@ -92,13 +93,13 @@ class TradeStatLog {
def run(){ def run(){
def citys = CcbCommonUtils.citys_all; def citys = CcbCommonUtils.citys_all;
def rets = [] def rets = []
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01" //def startDate = "2021-05-01"
def startDate = "2023-02-01" def startDate = "2023-03-01"
def date = "2023-03-01" def date = "2023-04-01"
def endDate = "2023-04-01" def endDate = "2023-05-01"
def newDate = "2023-05-01" def newDate = "2023-06-01"
for(def city : citys){ for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
cityRun(citys, startDate, endDate, cityCodeMap) // cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap) // regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
} }
@@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
citys = CcbCommonUtils.getRunningCitys(args) citys = CcbCommonUtils.getRunningCitys(args)
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
@@ -0,0 +1,261 @@
package com.fangjia.script.chongqing
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
class FumingTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new FumingTest();
task.run();
}
def run() {
def path = "D:\\数据\\富民\\富民测试500-v7.xlsx"
// match(path)
// addPrice(path)
addThirdPrice(path)
// addLastPrice(path)
// addFacLevel(path)
}
def addFacLevel(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['K'] && row['I']){
// def obj = findCommunitySummary(row['C'])
// if(obj){
// row['K'] = obj.consumption_rating
// }else{
row['K'] = getNearByDistrictLevel(row['C'], row['I'])
// }
}
})
}
/**
* 一公里小区数量
*/
def getNearByDistrictLevel(def id, def price){
BasicDBObject district = DbUtils.seaweed.findById(id);
def level = null
if(district){
List<DBObject> list = DbUtils.findAroundDistrict(district);
println id +" 共:"+list.size()+" "+ price
if(list!=null&&list.size()>0){
for(def obj : list){
def summary = findCommunitySummary(obj._id.toString())
if(summary && summary.avg_price && summary.consumption_rating){
def pre = java.lang.Math.abs((summary.avg_price - price)*100/price)
println summary.name +" "+pre +" "+ summary.avg_price
if(pre <= 10){
level = summary.consumption_rating
break;
}
}
}
}
}
return level;
}
def addLastPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Z'] ){
if(row['W']){
row['Z'] = row['W']
}else if(row['R']){
row['Z'] = row['R']
}else if(row['I']){
row['Z'] = row['I']
}
if(row['Z']){
row['J'] = (int)(row['Z']*row['B'].toString().toDouble())
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['N'] ){
def city = row['C']
def address = row['A']
def region = row['D']
def name = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['N'] = jsonObject.get("id")
row['O'] = jsonObject.get("city")
row['P'] = jsonObject.get("district")
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("propertyType")
row['S'] = jsonObject.get("address")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['C']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['D'] = dist.city
row['E'] = dist.region
row['F'] = dist.name
row['G'] = dist.estate_type2
row['H'] = dist.address
// row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['I'] = priceObj.avg_price
row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble())
row['R'] = priceObj.avg_price_2b
}
}
}
})
}
def addThirdPrice(path){
// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连']
def allMap = [:]
// for(def c : citys){
// def cityMap = findThirdPartyPrice(c)
// allMap << cityMap
// }
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['L'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['M'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['N'] = priceMap.get("anjuke_esf_price")
}
}
def map = findAvgPrice(row['C'], cityCodeMap.get(row['D']))
if(map){
row['O'] = map.avg_price
row['P'] = map.credit_value
}
})
}
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
if(d['C']){
def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
if(map){
d['K'] = map.avg_price
d['L'] = map.calc_type
// d['AI'] = map.credit_value
d['M'] = map.month
}
}
})
}
def findThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def findCommunitySummary(id){
def obj = null
def sql = "select * from community_summary where id='${id}'"
ElasticUtils.executeSqlQuery(sql, {
obj = it
})
return obj
}
}
@@ -40,19 +40,25 @@ class HailanTest {
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) { static main(args) {
def task = new HailanTest(); def task = new HailanTest();
task.run(); task.run();
} }
def run(){ def run(){
def inputfile = "D:\\数据\\海南\\统计.xlsx" def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
// addThirdPrice(inputfile)
// match(inputfile) // match(inputfile)
// tradeAvgPrice(inputfile)
// caluateTotalPrice(inputfile)
// addPrice(inputfile) // addPrice(inputfile)
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") // addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
// addHouse(inputfile) // addHouse(inputfile)
// stat(inputfile) // stat(inputfile)
hailanStat(inputfile) // hailanStat(inputfile)
addFzFjwPrice(inputfile)
} }
def addHouse(inputfile){ def addHouse(inputfile){
@@ -533,13 +539,13 @@ class HailanTest {
def match(path){ def match(path){
ExcelUtils.multiThreadingModify(path, { row -> ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){ if(row['A']){
String city = row['H']//RegionData.formatCity(row['B']) String city = "福州"//RegionData.formatCity(row['B'])
if(city){ if(city){
def address = row['C'] def address = row['A']
// def region = row['C'] // def region = row['C']
println "[match] city:"+city+" address:"+address println "[match] city:"+city+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E']) def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) { if(jsonObject != null && jsonObject.precise!=-1) {
row['I'] = jsonObject.get("district") row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("name") row['J'] = jsonObject.get("name")
@@ -554,6 +560,8 @@ class HailanTest {
}) })
} }
def stat(path){ def stat(path){
ExcelUtils.modify(path, { row -> ExcelUtils.modify(path, { row ->
if(row['L'] && !row['H']){ if(row['L'] && !row['H']){
@@ -700,4 +708,60 @@ class HailanTest {
} }
return prices; return prices;
} }
def tradeAvgPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){
def dist = CommonDatautils.findDistrictById(row['I'])
if(dist){
def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01");
row['J'] = (int)tradeAvgPrice
}
}
})
}
def addThirdPrice(path){
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate)
def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95)
}
}
})
}
def caluateTotalPrice(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['J']){
row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble())
}else if(row['K']){
row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble())
}
})
}
def addFzFjwPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['B']) {
def q = new BasicDBObject()
q.put("city", "福州")
q.put("region", row['A'])
q.put("name", row['B'])
q.put("month", 202304)
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if (obj) {
row['D'] = obj.avg_price
}
}
})
}
} }
@@ -26,7 +26,7 @@ class MingShenPocTest {
def run(){ def run(){
def path = "D:\\数据\\民生\\poc测试.xlsx" def path = "D:\\数据\\民生\\poc测试.xlsx"
//9级标准化 //9级标准化
//structureV2(path)//针对海南poc地址数据此接口不合适使用 structureV2(path)//针对海南poc地址数据此接口不合适使用
//省市区拆分 //省市区拆分
// ner(path)//城市,区域带市带区 // ner(path)//城市,区域带市带区
// match(path) // match(path)
@@ -34,7 +34,7 @@ class MingShenPocTest {
// addLatLng(path) // addLatLng(path)
// addAroundPrice(path) // addAroundPrice(path)
// addAuthor(path) // addAuthor(path)
addHouse(path) // addHouse(path)
} }
def structureV2(path){ def structureV2(path){
@@ -0,0 +1,197 @@
package com.fangjia.script.chongqing
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class RuiFenPocTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new RuiFenPocTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\瑞丰\\房产信息-地址V3-1.xlsx"
// match(inputfile)
// addValue(inputfile)
// addThirdPrice(inputfile)
lastPrice(inputfile)
}
def lastPrice(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(!row['X'] && row['S']){
def mod = java.lang.Math.abs((row['S'].toString().toDouble()-row['H'].toString().toDouble())*100/row['H'].toString().toDouble())
if(mod <= 10){
row['U'] = row['H']
}else{
row['U'] = row['S']
}
}else{
if(row['K']){
row['U'] = row['K']
}else{
row['U'] = row['H']
}
}
})
}
def addThirdPrice(path){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['L'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['M'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['N'] = priceMap.get("anjuke_esf_price")
}
}
def map = findAvgPrice(row['I'], cityCodeMap.get(row['D']))
if(map){
row['O'] = map.avg_price
row['P'] = map.credit_value
}
})
}
def addValue(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
def dist = CommonDatautils.findDistrictById(row['I'])
if(dist){
row["D"] = dist.city
row["E"] = dist.region
row["F"] = dist.name
row["G"] = dist.address
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['H'] = priceObj.avg_price
row['K'] = priceObj.avg_price_2b
}
}
})
}
def match(path) {
ExcelUtils.multiThreadingModify(path, { row ->
def city = "绍兴"
if (row['B'] && !row["D"]) {
def address = row['B']
println "[match] city:" + city + " address:" + address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null)
if(!row["D"]){
if (jsonObject != null && jsonObject.precise != -1) {
row["D"] = jsonObject.city
row["E"] = jsonObject.district
row["F"] = jsonObject.name
row["G"] = jsonObject.address
row["I"] = jsonObject.id
def priceObj = CommonDatautils.findDistrictPrice(jsonObject.city, jsonObject.district, jsonObject.name)
if(priceObj){
row['H'] = priceObj.avg_price
}
}
}else {
if (jsonObject != null && jsonObject.precise != -1) {
if(row['J']){
def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id
row['K'] = str
}else{
def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id
row['J'] = str
}
}
}
}
})
}
def findThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
}
@@ -1,11 +1,14 @@
package com.fangjia.script.commData package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper import groovy.json.JsonSlurper
import net.sf.json.JSONArray import net.sf.json.JSONArray
import net.sf.json.JSONObject import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId import org.bson.types.ObjectId
import com.fangjia.core.util.HttpRequest; import com.fangjia.core.util.HttpRequest;
@@ -23,6 +26,12 @@ import com.mongodb.DBCollection
import com.mongodb.DBCursor import com.mongodb.DBCursor
import com.mongodb.DBObject import com.mongodb.DBObject
import com.mongodb.QueryOperators import com.mongodb.QueryOperators
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
class CommonDatautils { class CommonDatautils {
@@ -73,6 +82,19 @@ class CommonDatautils {
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl) public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo); public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static DSCollection getHouseDataStoreConnection(){ public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) { if(house_dataStoreConnection != null) {
return house_dataStoreConnection return house_dataStoreConnection
@@ -230,6 +252,7 @@ class CommonDatautils {
q.put("visible", 0); q.put("visible", 0);
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
// q.put("name", "长滩美墅三期")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -781,7 +804,7 @@ class CommonDatautils {
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){ public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put("visible",0) q.put("status",0)
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
q.put("region", new BasicDBObject('$in', regions)) q.put("region", new BasicDBObject('$in', regions))
@@ -1001,7 +1024,7 @@ class CommonDatautils {
q.put("cat", cat); q.put("cat", cat);
q.put("region", region) q.put("region", region)
q.put("name", name); q.put("name", name);
q.put("status", 0) q.put("visible", 0)
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -3418,11 +3441,27 @@ class CommonDatautils {
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def sqmetreRentMap = [:] def sqmetreRentMap = [:]
for(def d : dbcursor){ for(def d : dbcursor){
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) // sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price)
} }
return sqmetreRentMap; return sqmetreRentMap;
} }
static def findAllSqmetreRentPrices(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("cat","district")
q.put("weekend", weekend)
def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000)
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def prices = []
for(def d : dbcursor){
prices << d.avg_price
}
return prices;
}
static def findNewSqmetreRent(def city, def region, def name){ static def findNewSqmetreRent(def city, def region, def name){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
@@ -3490,5 +3529,373 @@ class CommonDatautils {
} }
return obj return obj
} }
static def findDistrictAnalyStat(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q)
return analyData
}
static def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
.must(date_range).must(area_range)
if(cat){
queryBuilder.must(QueryBuilders.termQuery("cat", cat))
}
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
// println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
//// println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
// continue;
// }
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(seawater.height && seawater.height>60){
continue;
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("lianjia_esf_price", db.beike_esf_price)
}
}
return map
}
/**
* 根据经纬度获取区域
* @param city
* @param location
* @return
*/
static def getRegionByLocation(city, def location){
def json = GaoDeApiUtils.geocode(location);
def region = null;
if (json && json.info=="OK") {
def regeocode = json.regeocode
if(regeocode && regeocode.addressComponent){
region = regeocode.addressComponent.district
if(!region){
region = regeocode.addressComponent.township
}
}
}
if(region){
region = RegionData.formatRegion(city, region)
}
return region;
}
} }
@@ -0,0 +1,59 @@
package com.fangjia.script.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.mongodb.BasicDBObject
class MasterNameInAlias {
static main(args) {
def task = new MasterNameInAlias()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def allList = []
for(def city : citys){
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
def cityList = []
cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"]
for(def dist1 : disticts){
def distList = []
for(def dist2 : disticts){
if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){
if(dist2.alias && dist2.alias.contains(dist1.name)){
def distance = ''
if(dist1.lng && dist2.lng){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d);
}
distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance]
dist2.alias.remove(dist1.name)
def d = new BasicDBObject()
d.put("_id", dist2._id)
def udata = new BasicDBObject()
udata.put("alias", dist2.alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
}
}
}
if(distList){
cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', '']
cityList.addAll(distList)
// cityList << []
}
}
if(cityList){
allList.addAll(cityList)
println city +" "+cityList.size()
}
}
// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据")
}
}
@@ -35,7 +35,7 @@ class FjtResponseTimeAnaly extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
//run(); //run();
jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了?") jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了2?")
} }
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
@@ -82,20 +83,21 @@ class DistrictInfoExportBatch extends BaseBatch{
} }
} }
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
for(def dist : removeDists){ // for(def dist : removeDists){
def key = dist.city +"##"+dist.region+"##"+dist.name // def key = dist.city +"##"+dist.region+"##"+dist.name
if(!distKeys.contains(key)){ // if(!distKeys.contains(key)){
distKeys << key // distKeys << key
districts << dist // districts << dist
} // }
} // }
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
def startDate = DateUtils.getFirstDayOfMonth(lastDate); def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress(); def fzAddrMap = getFzAddress();
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据 //因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
saveOrUpdateFzDist(districts) // saveOrUpdateFzDist(districts)
remarkRemoveDist(removeDists)
def fzDistricts = getFzDistricts(city) def fzDistricts = getFzDistricts(city)
for(def rMap : fzDistricts){ for(def rMap : fzDistricts){
@@ -185,6 +187,15 @@ class DistrictInfoExportBatch extends BaseBatch{
} }
def remarkRemoveDist(removeDists){
for(def rd : removeDists){
def udate = new BasicDBObject()
udate.put("status",1)
udate.put("m_date", new Date())
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
}
}
def getFzDistricts(def city){ def getFzDistricts(def city){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
@@ -311,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def s = str.toString(); def s = str.toString();
index++; index++;
if(s && s!="null"){ if(s && s!="null"){
s = s.replace('"','\\"')
// if(s.toString().contains(",")){ // if(s.toString().contains(",")){
// s = csvHandlerStr(s.toString()) // s = csvHandlerStr(s.toString())
// //
@@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch {
if(weekendPriceObj.value_type!=1){ if(weekendPriceObj.value_type!=1){
if(dist.city == "¸£ÖÝ"){ if(dist.city == "¸£ÖÝ"){
if(Math.abs(mom)*100>3){ if(Math.abs(mom)*100>3){
def randMarkup = nextDouble(0, 3) if(dist.region != "Âíβ"){
if(mom < 0){ def randMarkup = nextDouble(0, 3)
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) if(mom < 0){
}else{ mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
mom = DataUtil.roundDouble(randMarkup/100, 4) }else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
} }
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
} }
}else{ }else{
if(Math.abs(mom)*100>10){ if(Math.abs(mom)*100>10){
@@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{
// sb.append(s+ "@|@") // sb.append(s+ "@|@")
// } // }
// } // }
sb.append(StringUtils.join(row.toArray(), "@|@")) def str = StringUtils.join(row.toArray(), "@|@")
str = str.replace('"','\\"')
sb.append(str)
sb.append(System.getProperty("line.separator")); sb.append(System.getProperty("line.separator"));
println index println index
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils import com.fangjia.script.gj.EsUtils
@@ -11,26 +12,91 @@ import com.mongodb.DBCollection
import org.apache.commons.io.FileUtils import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def datas = FileUtils.readLines("D:\\Êý¾Ý\\ÆäËü\\FJW_COMM_20221015.csv") ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
def rets = [] def city = RegionData.formatCity("余姚")
for(def d : datas){ d['D'] = city
def arrs = d.split("@") if(city){
rets << ["commId":arrs[0], "avg_price":arrs[4]] d['E'] = RegionData.formatRegion(city, d['C'])
}
for(def r : rets){
if(r.avg_price){
def udata = new BasicDBObject()
udata.put("date", 202210)
udata.put("avg", r.avg_price.toInteger())
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
}else{
println r.commId +" ²»´æÔÚ"
} }
// if(!d['D']){
// def citys = CommonDatautils.getCityByRegion(d['A'])
// if(citys && citys.size()==1){
// d['F'] = citys.get(0)
// }
// }
})
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
//def m = [:]
//for(def d : datas){
// m.put(d['B'], d)
//}
//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d->
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(dist){
// def ret = m.get(dist._id.toString())
// if(ret){
// d['E'] = ret['J']
// d['F'] = ret['N']
// }
// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS)
// if(dbo){
// if(dbo.beike_esf_price){
// d['G'] = dbo.beike_esf_price
// }else if(dbo.beike_esf_price){
// d['G'] = dbo.beike_esf_price
// }
//
// if(dbo.maitian_esf_price){
// d['H'] = dbo.maitian_esf_price
// }
// }
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd"))
// if(dbo){
// d['K'] = dbo.avg_price
// d['L'] = dbo.value_type
// }
//
// }
//})
def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", 202304)
return AVERAGE_STATISTICS.findOne(q)
} }
//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
//def rets = []
//for(def d : datas){
// def arrs = d.split("@")
// rets << ["commId":arrs[0], "avg_price":arrs[4]]
//}
//
//for(def r : rets){
// if(r.avg_price){
// def udata = new BasicDBObject()
// udata.put("date", 202210)
// udata.put("avg", r.avg_price.toInteger())
// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
// }else{
// println r.commId +" 不存在"
// }
//}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
@@ -29,8 +29,11 @@ class AddAvgpriceBatch {
} }
def test(){ def test(){
//发重算均价命令 def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"])
addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd')) for(def dist : districts){
//发重算均价命令
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
}
} }
def run(){ def run(){
@@ -96,17 +96,28 @@ class CityStatExport {
} }
def regionStat(){ def regionStat(){
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州']
def ret = [] def ret = []
ret << ["城市","区域","小区总量"] ret << ["城市","区域","小区总量"]
for(def city : citys){ for(def city : citys){
def regions = CommonDatautils.getRegionListByCity(city) def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){ for(def r : regions){
def row = []
row << city
row << r
def districts = CommonDatautils.findDistricts(city, r, null) def districts = CommonDatautils.findDistricts(city, r, null)
println city +" "+ r +" "+ districts.size() println city +" "+ r +" "+ districts.size()
row << CommonDatautils.queryCityRegionPrice(city, r, null)
def obj = CommonDatautils.findRegionPriceRate(city, r)
row << obj.price_coverage*100
row << districts.size()
ret << row
} }
} }
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县") ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县")
} }
@@ -272,6 +272,45 @@ class FiledStrand {
return null return null
} }
static def processName(name){
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
if (name.contains('(') && name.contains(')')){
def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1)
name = name.replace(rep_str, '')
}
if (name.contains('(') && name.contains(')')){
def rep_str = name.subsring(name.index('('),name.rfind(')')+1)
name = name.replace(rep_str, '')
}
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
name = name.replace(rep_str, '')
}
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
name = name.replace(rep_str, '')
}
if (name.find('(')!=-1 && name.find(')')!=-1){
def rep_str = name[name.index('('):name.rfind(')')+1]
name = name.replace(rep_str, '')
}
if (name.find('「')!=-1 && name.find('」')!=-1){
def rep_str = name[name.index('「'):name.rfind('」')+1]
name = name.replace(rep_str, '')
}
return name
}
static def processCarSite(car_site_count){ static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString()); def v = getValue(car_site_count.toString());
if(v){ if(v){
@@ -579,6 +618,12 @@ class FiledStrand {
}else{ }else{
return null; return null;
} }
}else if(estateType2=="写字楼"){
if(estateCharege>=0.35 && estateCharege<=60){
return estateCharege;
}else{
return null;
}
} }
}catch (Exception e){ }catch (Exception e){
println "estateCharege ת²»ÁË "+estateCharege println "estateCharege ת²»ÁË "+estateCharege
@@ -567,7 +567,7 @@ class OpenApiUtils {
} }
static def financialRating(city, address){ static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3" def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8"); String content = new HttpRequest().get(url, "utf-8");
if(content){ if(content){
def doc = new JsonSlurper().parseText(content) def doc = new JsonSlurper().parseText(content)
@@ -580,5 +580,79 @@ class OpenApiUtils {
} }
//源地址的完整性进行验证
static def addrVerify(city, address){
def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
// 省、市、区、街道、社区居委、POI补全
static def complete(city, address){
def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
//五级(省/市/区/街道/社区居委)解析、补全
static def division(city, address){
def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
//识别地址中省、市、区、道路、POI同时进行地址验证
static def structure(city, address){
def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
/**
* 招商银行测试
* @param city
* @param address
* @return
*/
static def cmbTest(city, address){
def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
} }
@@ -1,5 +1,6 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
@@ -69,29 +70,43 @@ import com.mongodb.DBObject
// //
// //
//}) //})
println "开始统计" //println "开始统计"
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1) //def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1)
def map = [:] //def map = [:]
for(def d : datas){ //for(def d : datas){
def prices = map.get(d['B']) // def prices = map.get(d['B'])
if(!prices){ // if(!prices){
prices = [] // prices = []
map.put(d['B'], prices) // map.put(d['B'], prices)
} // }
prices << d['G'] // prices << d['G']
}
def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
def rows = []
rows << ["区域","数量", "均价"]
regions.each{e->
def prices = map.get(e)
def price = (prices.sum()/prices.size()).toInteger()
rows << [e, prices.size(), price]
}
//map.each{region,prices->
// def price = (prices.sum()/prices.size()).toInteger()
// rows << [region, prices.size(), price]
//} //}
ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果") //
//def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
//def rows = []
//rows << ["区域","数量", "均价"]
//
//regions.each{e->
// def prices = map.get(e)
// def price = (prices.sum()/prices.size()).toInteger()
// rows << [e, prices.size(), price]
//}
////map.each{region,prices->
//// def price = (prices.sum()/prices.size()).toInteger()
//// rows << [region, prices.size(), price]
////}
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果")
//ExcelUtils.multiThreadingModify("D:\\数据\\其它\\城市等级.xlsx",{d->
// d['C'] = RegionData.formatCity(d['A'])
//})
DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo);
ExcelUtils.modify("D:\\数据\\其它\\城市等级.xlsx",{d->
def q = new BasicDBObject()
q.put("display", d['C'])
def udata = new BasicDBObject()
udata.put("new_city_grade",d['E'].toString().toDouble().toInteger())
SERVER_AREA_CONN.update(q, new BasicDBObject('$set', udata))
})
@@ -0,0 +1,49 @@
package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
if(d['A'] && !d['E']){
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
println d['A'] +" "+d['C']
// def total = findDatas(d['A'], d['B'], d['C'])
// d['D'] = total
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
d['E'] = c
}
})
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name.contains(name) && d.type.contains("住宅")){
total ++
}
})
return total;
}
def findDatas(def city, def region, def name){
def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'"
def total = 0
ElasticUtils.executeSqlQuery(sql, { d->
total = d.count
})
return total;
}
//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
// def region = RegionData.formatCity(d['A'])
// if(!region && d['A']){
// println d['A']
// }
//})
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-04-01" static def statDate = "2023-05-01"
static main(args) { static main(args) {
def task = new YouDaDataParser(); def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo // task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo // task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es // task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -76,24 +76,24 @@ MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") //def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") //def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
Calendar calendar = Calendar.getInstance() //Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate) //calendar.setTime(startDate)
def count = 0 //def count = 0
def hashSet = [] //def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] //def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
//
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> //OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result) // def result = JSONObject.fromObject(doc.res_map.result)
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] // def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item // data << item
hashSet << result // hashSet << result
} //}
//
//
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); ////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") //ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd") //def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
@@ -115,4 +115,27 @@ ExcelUtils.write("D:\\
// //
// //
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); ////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") //ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230524", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]]
OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode,
doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber]
data << item
hashSet << result
}
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
@@ -0,0 +1,207 @@
package com.fangjia.script.gj.data
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 按区域统计覆盖率
*/
class XiamenStat {
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
public static final def DISTRICT_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
]
static main(args) {
def task = new XiamenStat();
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d->
if(d['D'] && d['D']=="全市"){
def dbo = CommonDatautils.findCityInfo(d['C'].replace("市",""))
if(dbo){
def detail = dbo.district_detail
if(detail){
d['E'] = dbo.existing_district_count//数据量
d['F'] = detail.township*100// 乡镇街道
d['G'] = detail.address*100// 地址
d['H'] = detail.addrAlias*100// 地址别名
d['I'] = detail.name*100// 名称
d['J'] = detail.alias*100// 别名
d['K'] = detail.alias*100// 映射别名
d['L'] = detail.lat2*100// 经纬度(百度)
d['M'] = detail.lat*100// 经纬度(高德)
d['N'] = detail.estateType2*100// 物业类型
d['O'] = detail.buildType*100// 建筑类型
d['P'] = detail.streetCommunity*100// 街道社区
d['Q'] = detail.committee*100// 居委会
d['R'] = detail.businessQuarter*100// 商圈
d['S'] = detail.schools*100// 对口小学
d['T'] = detail.kindergartens*100// 对口幼儿园
d['U'] = detail.loudongCount*100// 楼栋总数
d['V'] = detail.householdCount*100// 户数
d['W'] = detail.estateCharge*100// 物业费
d['X'] = detail.carSiteCount*100// 停车位
d['Y'] = ""// 电梯
d['Z'] = detail.propertyLife*100// 产权年限
d['AA'] = detail.houseProperty*100// 产权性质
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
d['AC'] = detail.landProperty*100// 小区性质
d['AD'] = detail.finishBuildingDate*100// 竣工时间
d['AE'] = detail.openDate*100// 交房时间
d['AF'] = detail.allArea*100// 占地面积
d['AG'] = detail.buildArea*100// 建筑面积
d['AH'] = detail.cubagePercent*100// 容积率
d['AI'] = detail.virescencePercent*100// 绿化率
d['AJ'] = detail.mainType*100// 主力户型
d['AK'] = detail.mainTypeArea*100// 主力户型面积
d['AL'] = detail.developer*100// 开发商
d['AM'] = detail.estateCompany*100// 物业公司
d['AN'] = dbo.price_coverage*100// 小区当前均价
}
}
}else{
def fields = DISTRICT_FIELDS_MAP.keySet().asList()
def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields)
def detail = districtDetail(regionDistricts.size(), regionDistricts);
if(detail){
d['E'] = regionDistricts.size()//数据量
d['F'] = detail.township*100// 乡镇街道
d['G'] = detail.address*100// 地址
d['H'] = detail.addrAlias*100// 地址别名
d['I'] = detail.name*100// 名称
d['J'] = detail.alias*100// 别名
d['K'] = detail.alias*100// 映射别名
d['L'] = detail.lat2*100// 经纬度(百度)
d['M'] = detail.lat*100// 经纬度(高德)
d['N'] = detail.estateType2*100// 物业类型
d['O'] = detail.buildType*100// 建筑类型
d['P'] = detail.streetCommunity*100// 街道社区
d['Q'] = detail.committee*100// 居委会
d['R'] = detail.businessQuarter*100// 商圈
d['S'] = detail.schools*100// 对口小学
d['T'] = detail.kindergartens*100// 对口幼儿园
d['U'] = detail.loudongCount*100// 楼栋总数
d['V'] = detail.householdCount*100// 户数
d['W'] = detail.estateCharge*100// 物业费
d['X'] = detail.carSiteCount*100// 停车位
d['Y'] = ""// 电梯
d['Z'] = detail.propertyLife*100// 产权年限
d['AA'] = detail.houseProperty*100// 产权性质
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
d['AC'] = detail.landProperty*100 // 小区性质
d['AD'] = detail.finishBuildingDate*100// 竣工时间
d['AE'] = detail.openDate*100// 交房时间
d['AF'] = detail.allArea*100// 占地面积
d['AG'] = detail.buildArea*100// 建筑面积
d['AH'] = detail.cubagePercent*100// 容积率
d['AI'] = detail.virescencePercent*100// 绿化率
d['AJ'] = detail.mainType*100// 主力户型
d['AK'] = detail.mainTypeArea*100// 主力户型面积
d['AL'] = detail.developer*100// 开发商
d['AM'] = detail.estateCompany*100// 物业公司
// d['AN'] = ""// 小区当前均价
}
}
})
}
def districtDetail(def disCount, def regionDistricts) {
def countMap = [:]
for(def dbo : regionDistricts) {
DISTRICT_FIELDS_MAP.each() {key,skey->
if (isValid(dbo.get(key))) {
if(countMap.containsKey(key)) {
countMap.put(key, countMap.get(key) + 1)
}else {
countMap.put(key, 1)
}
}
}
}
def districtDetail = [:];
DISTRICT_FIELDS_MAP.each() {key,skey->
if(countMap.containsKey(key)) {
districtDetail.put(skey, roundDouble(countMap.get(key), disCount))
}else {
districtDetail.put(skey, 0)
}
}
if(countMap.containsKey("lat") && countMap.containsKey("lng")) {
districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount));
}else {
districtDetail.put('latLng', 0);
}
if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) {
districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount));
}else {
districtDetail.put('latLng2', 0);
}
return districtDetail;
}
def CONTAINS_KWS = ["...", "暂无", "待定", "未知"]
def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."]
def isValid(def value) {
if (!value) {
return false;
}
if (value instanceof String) {
value = value.trim()
for(String ck : CONTAINS_KWS) {
if(value.contains(ck)) {
return false;
}
}
for(String ek : EQUALS_KWS) {
if(ek.equals(value)) {
return false;
}
}
} else if (value instanceof Number) {
if (value <= 0) {
return false;
}
} else if (value instanceof Collection || value instanceof Map) {
if (value.size() == 0) {
return false;
}
}
return true;
}
double roundDouble(int a, int base) {
if(a == 0 || base == 0) {
return 0
}
return ((double)a / (double)base * 10000.0).toInteger() / 10000.0
}
}
@@ -0,0 +1,117 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 通过房源更改电梯
*/
@BatchOwner(Watcher.GaoJuan)
class AddDistrictElevatorBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
static main(args) {
runBatch(new AddDistrictElevatorBatch(), args)
}
@Override
void process(Object args) {
run()
}
def run(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def dists = CommonDatautils.findAllDistricts(c, "district", ["city","region","name","alias", "build_type", "estate_type2", "is_elevator"])
addEvevatorProcess(c, dists)
}
}
def addEvevatorProcess(c, dists){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
println "小区量:"+dists.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<dists.size();i++){
final def dist = dists[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(!dist.is_elevator){
def houses = CommonDatautils.findHouses(dist.city, dist.region, null, 20200101, 20230701, dist.name, dist.alias, dist.estate_type2)
def haveHouses = []
for(def h : houses){
if(h.height){
haveHouses << h
}
}
def newHouses = analyElevator(haveHouses, dist.estate_type2)
if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size()
BasicDBObject udata = new BasicDBObject()
if(dist.estate_type2 == "别墅"){
udata.put("is_elevator", false)
}else{
udata.put("is_elevator", true)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
rets << dist
}else{
if(dist.build_type && dist.build_type.contains("高层")){
BasicDBObject udata = new BasicDBObject('is_elevator',true)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
rets << dist
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
println c +" "+ rets.size()
// return retMap
}
def analyElevator(houses, estateType){
def heights = []
for(def h : houses){
if(estateType=="别墅"){
if(h.height && h.height<=4) {
heights << h.height
}
}else{
if(h.height && h.height>=10){
heights << h.height
}
}
}
return heights
}
}
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern import java.util.regex.Pattern
/** /**
* 小区字段入库(自动入库) * 小区字段填充入库(自动入库)
*/ */
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class DistrictFieldAddBatch extends BaseBatch{ class DistrictFieldAddBatch extends BaseBatch{
@@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
def citys = ["漳州"]//CommonDatautils.findAllCity().reverse() def citys = CommonDatautils.findAllCity().reverse()
def analysRets = [] def analysRets = []
@@ -95,7 +95,7 @@ class FangjiaClientUtils {
try { try {
if(address){ if(address){
address = address.replace("{","") address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
if(city){ if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8") url = url +"&city="+URLEncoder.encode(city,"UTF-8")
} }
@@ -0,0 +1,79 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 写字楼字段清洗入库(自动入库)
*/
@BatchOwner(Watcher.GaoJuan)
class OfficeFieldCleanBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new OfficeFieldCleanBatch(), args)
}
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
for(def c : citys){
def offices = findAllOffice(c)
strandField(offices)
}
}
def strandField(offices){
def fields = ["property_fee","build_area","","","","","","","","","",""]
for(def office : offices){
fields.each{
def newVal = null
if(it == "property_fee"){
newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼")
newVal = newVal.toString()
}
if(it =="all_area" || it =="build_area"){
newVal = FiledStrand.processArea(office.get(it))
if(newVal && newVal > 5000){
newVal = newVal.toString()
}
}
if(it =="name"){
newVal = FiledStrand.processName(office.get(it))
if(newVal){
newVal = newVal.toString()
}
}
}
}
}
def findAllOffice(city){
def sql = "select * from office where city='${city}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
})
return rets
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
@@ -15,7 +16,48 @@ class SameDistName {
static main(args) { static main(args) {
def task = new SameDistName() def task = new SameDistName()
task.run() // task.run()
//地址和地址别名完全一致
// task.addressPocess()
}
def addressPocess(){
def citys = CommonDatautils.findAllCity();
for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"])
if(dists){
def index = 0
for(def dist : dists){
if(dist.address){
// if(dist.address.toString().endsWith("弄")){
// dist.address = dist.address.toString().replace("弄","号")
// }
if(dist.addr_alias && dist.addr_alias.contains(dist.address)){
def new_address_alias = []
for(def addr : dist.addr_alias){
if(!new_address_alias.contains(addr) && addr!=dist.address){
new_address_alias << addr
}
}
index ++
println "[${index}] "+dist.city +" "+dist.name+" 旧:"+dist.addr_alias+" 新:"+ new_address_alias
def d = new BasicDBObject()
d.put("_id", dist._id)
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("addr_alias", new_address_alias)
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
}
}
}
}
}
} }
def run(){ def run(){
@@ -144,5 +144,8 @@ class SeaweedDataMatchProcess {
} }
public static void main(def agrs){
doMatch("上海", "静安", "金家巷西弄", null)
}
} }
@@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&
stat() stat()
def stat(){ def stat(){
def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1) def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1)
def map=[:] def map=[:]
for(def d : datas){ for(def d : datas){
def count = map.get(d['C']) def count = map.get(d['C'])
@@ -69,8 +69,8 @@ def findLogs(){
def findFangjiaLogs(){ def findFangjiaLogs(){
def startDate = DateUtil.parseDate("20230329", "yyyyMMdd") def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230429", "yyyyMMdd") def endDate = DateUtil.parseDate("20230531", "yyyyMMdd")
Calendar calendar = Calendar.getInstance() Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate) calendar.setTime(startDate)
def list = [] def list = []
@@ -84,14 +84,21 @@ def findFangjiaLogs(){
QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false); QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") // def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def q = QueryBuilders.fuzzyQuery("request", "/enquire")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size() println houses.size()
if (houses) { if (houses) {
for (def d : houses) { for (def d : houses) {
def reqStr = d.request.toString() def reqStr = d.request.toString()
if(!reqStr.contains("/enquire")){
continue
}
def mid = null def mid = null
if (reqStr.contains("mid=")) { if (reqStr.contains("mid=")) {
@@ -107,5 +114,5 @@ def findFangjiaLogs(){
startDate = nextDate startDate = nextDate
} }
ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志") ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志")
} }
@@ -0,0 +1,232 @@
package com.fangjia.script.gj.es
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils
import org.apache.lucene.queryparser.classic.QueryParser
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class HouseAnaly {
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new HouseAnaly()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def allRets = []
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"])
def rets = processing(districts)
println "[${city}] 写文件 "+rets.size()
allRets.addAll(rets)
}
ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析")
}
def processing(dists){
def rets = new CopyOnWriteArrayList()
rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(dist.alias && dist.alias.size()>=1){
def ret = analy(dist)
if(ret){
rets.addAll(ret)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def analy(dist){
def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是");
def alias = dist.alias
for(def aliasName : alias){
if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){
def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否");
if(aliaRet){
masterRet.addAll(aliaRet)
}
}
}
if(masterRet.size()>=2){
def masterAvgPrice = 0
def masterMidPrice = 0
def back = false
for(def row : masterRet){
if(row.get(4)=="是"){
masterAvgPrice = row.get(8)
masterMidPrice = row.get(9)
}else{
def pre1 = 0
def pre2 = 0
if(masterAvgPrice>0){
pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice)
}
if(masterMidPrice>0){
pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice)
}
if(pre1>=20 || pre2>=20){
back = true
}
}
}
if(back){
return masterRet
}
}
return null;
}
//小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量
def runAnalying(id, city, region, name, flag){
def houses = findHouse(city, region, name);
def rets = []
if(houses && houses.size()>=5){
def prices = []
for(def house : houses){
prices << house.avg_price
}
prices = prices.sort()
def average = (int)(prices.sum() / prices.size())
def midPrice = getMidPrice(prices)
def price10 = null
def price90 = null
if(houses.size()>=10){
def postion = Math.round(houses.size()/10)
price10 = prices.get((int)(postion-1))
price90 = prices.get((int)(houses.size()-postion-1))
}
def countMap = classType(houses)
// def pre = java.lang.Math.abs((midPrice-average)*100/average)
// if(pre>=20){
rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")]
// }
}
return rets
}
def classType(houses){
def district = 0
def villa = 0
def business = 0
for(def h : houses){
if(h.estate_type){
if(h.estate_type == "别墅"){
villa = villa + 1
continue
}else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){
business = business + 1
continue
}
}
def valla = isVilla(h.area, h.name, h.height, h.room)
if(valla){
villa = villa + 1
continue
}
district = district + 1
}
return ["district":district, "villa":villa, "business":business]
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getMidPrice(def prices){
def mid = prices.size()/2
if(mid > (int)mid){
return prices.get((int)mid)
}else{
def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1))
return (int)(total/2)
}
return null
}
def findHouse(def city, def region, def name){
def houses = []
try{
name = escapeExprSpecialWord(name)
String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'"
ElasticUtils.executeSqlQuery(sql,{ d->
def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type]
houses << r
// println r
})
}catch(Exception e){
e.printStackTrace()
}
return houses
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
@@ -21,13 +21,23 @@ class GaoDeMapSearchTest {
static main(args) { static main(args) {
def task = new GaoDeMapSearchTest(); def task = new GaoDeMapSearchTest();
def path = "D:\\数据\\光大\\光大测试-V2.xlsx" def path = "D:\\数据\\富民\\富民测试500.xlsx"
// task.structureV2(path) // task.structureV2(path)
// task.match(path) // task.match(path)
// task.mapSearch(path) // task.mapSearch(path)
task.baiduMapSearch(path) // task.baiduMapSearch(path)
task.checkTrue(path) // task.checkTrue(path)
task.addFinancialLevel(path)
}
def addFinancialLevel(path){
ExcelUtils.multiThreadingModify(path, { row ->
def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A'])
if(results){
row['V'] = results.consumptionRating
}
});
} }
def checkTrue(path){ def checkTrue(path){
@@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
//广州,佛山 重庆 //广州,佛山 重庆
static main(args) { static main(args) {
def task = new BeiKeCrawlByChrome(); def task = new BeiKeCrawlByChrome();
@@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome {
println city +" 已运行" println city +" 已运行"
continue continue
} }
def newCity = RegionData.formatCity(city)
if(!newCity){
if(city=="昆山"){
newCity = "苏州"
}
println city +" 无法格式化"
}
def url = m[1] def url = m[1]
def newUrl = url+"/chengjiao/" def newUrl = url+"/chengjiao/"
driver.get(newUrl) driver.get(newUrl)
@@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome {
def href = url+ r.attr("href") def href = url+ r.attr("href")
println "开始抓取【${city}】【${regionText}】【${href}】" println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText) if(city == "昆山"){
regionText = "昆山"
}
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println url +" 详情链接:"+detailUrls.size() println url +" 详情链接:"+detailUrls.size()
} }
} }
@@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){ if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl detailUrls << durl
parse(city, e, region) def dbo = findTrade(durl , "贝壳找房")
if (!dbo) {
parse(city, e, region, driver)
}
}else{ }else{
stop = true; stop = true;
break; break;
@@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome {
return detailUrls return detailUrls
} }
def parse(city,e, region){ def parse(city,e, region, driver){
try { try {
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
def title = e.select(".title a").text() def title = e.select(".title a").text()
if (title) { if (title) {
def detail = new BasicDBObject() def detail = new BasicDBObject()
def arrs = title.split(" ") def arrs = title.split(" ")
if(title.contains("车位")){
detail.type = "车位"
}
detail.city = city detail.city = city
detail.district_name = arrs[0] detail.district_name = arrs[0]
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name) detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
@@ -266,28 +289,48 @@ public class BeiKeCrawlByChrome {
} }
detail.source = "贝壳找房" detail.source = "贝壳找房"
if(detail.total_price && detail.avg_price){ def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
def detailUrl = e.select(".title a").attr("href") def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
detail.url = detailUrl
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date() def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
detail.m_date = new Date() if(liftStr){
println detail detail.lift = DataFormatUtils.parseLift(liftStr)
DEAL_PRICE_COLL.save(detail) detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡",""))
detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","")
detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限",""))
if(detail.total_price && detail.avg_price){
detail.url = detailUrl
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
detail.floor_info = floorStr
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorInfo && floorInfo.isNumber()) {
detail.height = floorInfo.toInteger()
}
} }
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
} }
} }
}catch(Exception ex){ }catch(Exception ex){
@@ -392,16 +435,16 @@ public class BeiKeCrawlByChrome {
} }
def findTrade(def detail){ def findTrade(def url, source){
def q = new BasicDBObject() def q = new BasicDBObject()
// q.put("city", detail.city) // q.put("city", detail.city)
// q.put("region", detail.region) // q.put("region", detail.region)
// q.put("district_name", detail.district_name) // q.put("district_name", detail.district_name)
// q.put("area", detail.area) // q.put("area", detail.area)
// q.put("total_price", detail.total_price) // q.put("total_price", detail.total_price)
q.put("url", detail.url) q.put("url", url)
// q.put("trade_date", detail.trade_date) // q.put("trade_date", detail.trade_date)
q.put("source", detail.source) q.put("source", source)
def dbo = DEAL_PRICE_COLL.findOne(q) def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo return dbo
} }
@@ -448,10 +491,13 @@ public class BeiKeCrawlByChrome {
btn.submit(); // 点击行为; btn.submit(); // 点击行为;
Thread.sleep(10000) Thread.sleep(10000)
}catch(Exception e){ }catch(Exception e){
Thread.sleep(10000) Thread.sleep(15000)
} }
return driver return driver
} }
} }
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.paimai
class DataFormatUtils {
static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000]
static def parseLift(def str){
if(str && str.contains("梯")){
def lift = str.toString().substring(0, str.toString().indexOf("梯"))
if(lift){
def val = chineseIntegerPart2Digit(lift)
if(val){
return (int)val
}
}
}
return null
}
static def parseHousehold(def str){
if(str && str.contains("梯") && str.contains("户")){
def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户"))
def val = chineseIntegerPart2Digit(household)
if(val){
return (int)val
}
}
return null
}
static def chineseIntegerPart2Digit(String chineseDigit)
{
Double finalDigit = 0.0;
Integer r = 1;
for(int i = chineseDigit.length()-1; i>=0;i--)
{
Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i)));
if( val >= 10 && i == 0 )
{
if (val > r)
{
r = val;
finalDigit = finalDigit +val;
}
else
{
r = r * val;
}
}
else if(val >= 10)
{
if (val > r)
{
r = val;
}
else
{
r = r * val;
}
}
else
{
finalDigit = finalDigit + r*val;
}
}
return finalDigit;
}
static String getValue(String s) {
if (s && s.contains("暂无")) {
return null
}
return s
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.paimai package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.ccb.FileUtils
@@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM")
static main(args) { static main(args) {
def task = new LianjiaCrawlByChrome(); def task = new LianjiaCrawlByChrome();
@@ -196,9 +197,22 @@ public class LianjiaCrawlByChrome {
println city +" 已运行" println city +" 已运行"
continue continue
} }
def newCity = RegionData.formatCity(city)
if(!newCity){
if(city=="昆山"){
newCity = "苏州"
}
println city +" 无法格式化"
}
def cityUrl = ret[1] def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/" def newUrl = cityUrl+"chengjiao/"
driver.get(newUrl) try{
driver.get(newUrl)
}catch(Exception e){
println newUrl +" 链接有问题!"
continue
}
//def detailUrls = getPageUrls(driver, cityUrl) //def detailUrls = getPageUrls(driver, cityUrl)
Thread.sleep(10000) Thread.sleep(10000)
def doc = checkDocument(driver, newUrl); def doc = checkDocument(driver, newUrl);
@@ -208,9 +222,11 @@ public class LianjiaCrawlByChrome {
for(def r : regions){ for(def r : regions){
def regionText = r.text() def regionText = r.text()
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href") def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
if(city == "昆山"){
regionText = "昆山"
}
println "开始抓取【${city}】【${regionText}】【${href}】" println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText) def detailUrls = getPageDeail(driver, newCity, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size() println cityUrl +" 详情链接:"+detailUrls.size()
} }
} }
@@ -282,13 +298,21 @@ public class LianjiaCrawlByChrome {
def dealDate = e.select(".dealDate").text() def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){ if(dealDate){
detailUrls << durl if(dealDate.getTime()>=endDate.getTime()){
parse(city, e, regionText) detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver)
}
}else{
stop = true;
break;
}
}else{ }else{
stop = true; println url +" 成交时间有误!"
break;
} }
} }
if(page>=100){ if(page>=100){
stop = true; stop = true;
@@ -302,85 +326,116 @@ public class LianjiaCrawlByChrome {
return detailUrls return detailUrls
} }
def parse(city,e, regionText){ def parse(city,e, regionText, driver){
try { try {
def title = e.select(".title a").text() def price = e.select(".totalPrice .number").text()
if (title) { if(price){
def detail = new BasicDBObject() def detailUrl = e.select(".title a").attr("href")
def arrs = title.split(" ") driver.get(detailUrl)
detail.city = city def detailDoc = checkDocument(driver, detailUrl);
detail.district_name = arrs[0] def title = e.select(".title a").text()
if(regionText){ if (title) {
detail.region = regionText def detail = new BasicDBObject()
}else{ def arrs = title.split(" ")
detail.region = findRegionByThirdPary(detail.city, detail.district_name) if(title.contains("车位")){
} detail.type = "车位"
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
} }
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { detail.city = city
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) detail.district_name = arrs[0]
if(hall && hall.isNumber()){ if(regionText){
detail.hall = hall.toInteger() detail.region = regionText
} }else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
} }
} if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
if("车位"!=roomStr){ }
if(arrs && arrs.size()>=3){ if(!detail.region){
def areaStr = arrs[2].replace("平米", "") def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
if (areaStr) { str = str.substring(0, str.indexOf("',"))
detail.area = areaStr.toDouble() def locArrs = str.split(",")
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
if(gmap){
detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat)
} }
} }
def price = e.select(".totalPrice .number").text() def roomStr = arrs[1]
if (price && price.isNumber()) { if (roomStr) {
detail.total_price = (int)(price.toDouble() * 10000) if (roomStr.indexOf("室") > 0) {
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
}
} }
if (detail.total_price && detail.area) { if("车位"!=roomStr){
detail.avg_price = (int) (detail.total_price / detail.area) if(arrs && arrs.size()>=3){
} def areaStr = arrs[2].replace("平米", "")
def tradeStr = e.select(".dealDate").text() if (areaStr) {
if (tradeStr) { detail.area = areaStr.toDouble()
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") }
} }
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
if(detail.total_price && detail.avg_price){
def dbo = findTrade(detail) if (price && price.isNumber()) {
if (!dbo) { detail.total_price = (int)(price.toDouble() * 10000)
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = detailUrl
def detailDiv = JQueryUtils.select(".introContent", detailDoc)
def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层",""))
if(detail.total_price && detail.avg_price){
def houseInfo = e.select(".houseInfo").text() def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|") def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0] detail.direction = houseInfos[0]
detail.fitment = houseInfos[1] detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
detail.floor_info = floorStr
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) { if (floorStr2 && floorStr2.isNumber()) {
detail.height = floorStr.toInteger() detail.height = floorStr2.toInteger()
} }
} }
detail.create_date = new Date() detail.create_date = new Date()
detail.m_date = new Date() detail.m_date = new Date()
println detail println detail
DEAL_PRICE_COLL.save(detail) DEAL_PRICE_COLL.save(detail)
} }
} }
} }
} }
}catch(Exception ex){ }catch(Exception ex){
@@ -484,16 +539,16 @@ public class LianjiaCrawlByChrome {
} }
def findTrade(def detail){ def findTrade(def url, source){
def q = new BasicDBObject() def q = new BasicDBObject()
// q.put("city", detail.city) // q.put("city", detail.city)
// q.put("region", detail.region) // q.put("region", detail.region)
// q.put("district_name", detail.district_name) // q.put("district_name", detail.district_name)
// q.put("area", detail.area) // q.put("area", detail.area)
// q.put("total_price", detail.total_price) // q.put("total_price", detail.total_price)
q.put("url", url)
// q.put("trade_date", detail.trade_date) // q.put("trade_date", detail.trade_date)
q.put("url", detail.url) q.put("source", source)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q) def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo return dbo
} }
@@ -530,12 +585,14 @@ public class LianjiaCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995"); // elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemPassword.sendKeys("12qwaszx"); elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为; btn.submit(); // 点击行为;
Thread.sleep(10000) Thread.sleep(15000)
}catch(Exception e){ }catch(Exception e){
Thread.sleep(10000) Thread.sleep(10000)
} }
@@ -0,0 +1,36 @@
package com.fangjia.script.gj.region
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 格式化其它网站区域
*/
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl)
def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1)
for(def d : datas){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("recognizeCity", d['F'])
q.put("recognizeRegion", d['A'])
def count = SERVER_AREA_COLL.count(q)
if(!count){
def obj = new BasicDBObject()
obj.put("city", d['A'])
obj.put("recognizeRegion", d['A'])
obj.put("recognizeCity", d['F'])
obj.put("createdDate", new Date())
SERVER_AREA_COLL.save(obj)
}else{
println d['A']+" "+ d['F'] +" 已存在"
}
}
@@ -58,8 +58,9 @@ class CZDataBatch extends BaseBatch{
def commids = findAllJnBankDatas(city) def commids = findAllJnBankDatas(city)
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
districtRun(districts, statMap, commids);
FzDataUtils.removeNoUpdate(city, districts) FzDataUtils.removeNoUpdate(city, districts)
districtRun(districts, statMap, commids);
def shops = CommonDatautils.findAllDistricts(city, "shop", fields); def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
shops = mergeShop(shops); shops = mergeShop(shops);
@@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
if(dists){ if(dists){
for(def d : dists){ for(def d : dists){
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) def distObj = null
if(d.comm_id){
distObj = CommonDatautils.findDistrictById(d.comm_id)
}else{
distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
}
if(distObj && distObj._id){ if(distObj && distObj._id){
FzDataUtils.converDistrict(distObj, null, null); if(d.type && d.type==1){
FzDataUtils.removeById(d.comm_id)
}else{
FzDataUtils.converDistrict(distObj, null, null);
}
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
updateTaskStatus(d.id, 2); updateTaskStatus(d.id, 2);
} }
@@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("status", 5);//未处理 q.put("status", 5);//未处理
TASK_COLL.find(q).each {d-> TASK_COLL.find(q).each {d->
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name] rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id]
} }
return rets; return rets;
} }
@@ -119,6 +119,13 @@ class FzDataUtils {
} }
} }
static def removeById(id){
def del_data = new BasicDBObject()
del_data.put("num_id", id)
DIST_COLL.remove(del_data)
}
static def saveDistrict(def m){ static def saveDistrict(def m){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put("num_id",m.num_id) q.put("num_id",m.num_id)
@@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias-> cityMap.each{city,alias->
println "역迦속潼±${city}×" println "역迦속潼±${city}×"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) // if(city == "¸£ÖÝ"){
districtMap.putAll(detailMap) def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
} }
while(flag){ while(flag){
@@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{
} }
} }
} }
if(!dbo){ if(!dbo && isStandardRoad(copyObj.address)){
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
if(result.size()==1){ if(result.size()==1){
dbo = result[0]; dbo = result[0];
@@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){ def findAllNoMatchData(){
DBObject qdata = new BasicDBObject(); DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE) qdata.put("s_date", SHORT_DATE)
// qdata.put("city","¸£ÖÝ") qdata.put("city","¸£ÖÝ")
// qdata.put("source","갛앙와") // qdata.put("source","갛앙와")
// qdata.put("name",Pattern.compile("묠소酪.*")) // qdata.put("name",Pattern.compile("묠소酪.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
@@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
} }
@Override @Override
void process(Object args) { void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date()); def weekend = DateUtils.getFirstDayOfWeek(new Date());
@@ -237,7 +237,7 @@ public class DbUtils{
def lng = district.lng / 10000000000.0 def lng = district.lng / 10000000000.0
sdq.put("sd.point", lat + "," + lng); sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng"); sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0)); sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0));
query.append('$sd', sdq) query.append('$sd', sdq)
def keys = new BasicDBObject() def keys = new BasicDBObject()