Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
zhaojixiang@fangjia.com
2023-07-06 13:45:49 +08:00
55 changed files with 2668 additions and 296 deletions
@@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ServerAreaMap
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* Created by dxy on 2016/12/15.
*/
@@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch {
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def DISTRICT_STAT_MAP = [:]
public void process(Object args) {
def cities = ServerAreaMap.findAllCities()
def cities = ["深圳"]//ServerAreaMap.findAllCities()
for(String city : cities) {
DISTRICT_STAT_MAP.clear()
Map<String, List<String>> regions = RegionData.getAllRegions(city)
regions.each {region, alias->
processRegionRank(city, region)
}
processCityRank(city)
// DISTRICT_STAT_MAP.clear()
// Map<String, List<String>> regions = RegionData.getAllRegions(city)
// regions.each {region, alias->
// processRegionRank(city, region)
// }
// processCityRank(city)
processCityRentRank(city)
}
}
@@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch {
return ret
}
def processCityRentRank(city){
def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd")
def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth)
def prices = []
distRentsMap.each{key, price->
if(!prices.contains(price)){
prices << price
}
}
// prices = prices.unique()
prices = prices.sort().reverse()
def totalCount = new AtomicLong()
def keys = distRentsMap.keySet().asList()
final CountDownLatch latch = new CountDownLatch(keys.size())
// for(def i=0;i<keys.size();i++){
// final def key = keys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
// totalCount.addAndGet(1)
// upateDistrict(key, distRentsMap, prices)
// println totalCount.get()+" "+ key
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// })
// }
// latch.await()
def index = 0
distRentsMap.each{key, price->
index ++
def arrs = key.split("##");
double percentile = calculatePercentile(prices, price);
DBObject q = new BasicDBObject("city", arrs[0])
q.append("region", arrs[1])
q.append("district_name", arrs[2])
DBObject udata = new BasicDBObject()
udata.put("rent_city_rank", (int)percentile);
udata.put("m_date", new Date())
DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
println index +" "+ arrs[2]
}
}
def upateDistrict(key, distRentsMap, prices){
def arrs = key.split("##");
def price = distRentsMap.get(key)
double percentile = calculatePercentile(prices, price);
DBObject q = new BasicDBObject("city", arrs[0])
q.append("region", arrs[1])
q.append("district_name", arrs[2])
DBObject udata = new BasicDBObject()
udata.put("rent_city_rank", (int)percentile);
udata.put("m_date", new Date())
// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
public static int calculatePercentile(def values, def targetValue) {
// 找出数字在List中的排名
def rank = values.indexOf(targetValue) + 1
// 计算百分位数
def percentile = (int)((rank / values.size()) * 100)
return percentile;
}
static main(args) {
runBatch(new DistrictStatRankBatch(), args)
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -51,7 +51,19 @@ for(def d : datas){
map.region = d['B']
def roomStr = d['E']
if(roomStr){
map.room = roomStr.toString().toDouble().toInteger()
if(roomStr.contains("-")){
def arrs = roomStr.split("-")
try{
map.room = arrs[0].toString().toInteger()
}catch(Exception e){
println "room error"
}
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
}else{
map.room = roomStr.toString().toDouble().toInteger()
}
}
map.area = d['F'].toString().toDouble()
@@ -59,7 +71,13 @@ for(def d : datas){
map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
def trade_date = d['J']
map.trade_date = trade_date
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
map.source = "深房家族"
map.create_date = new Date()
map.m_date = new Date()
@@ -77,7 +77,6 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
delPgData()
saveToPgSql(rows)
@@ -66,7 +66,8 @@ class BigdataDealBatch extends BaseBatch{
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
// for(def city : citys){
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
@@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
final CountDownLatch latch = new CountDownLatch(trades.size())
def final AtomicInteger count_ = new AtomicInteger(0)
@@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
}
}
if(rMap){
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}
}
@@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
@@ -428,7 +428,7 @@ class CcbCommonUtils {
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
@@ -28,9 +28,11 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
task.deleteThirdPartyData()
// task.deleteThirdPartyData()//删除第三方
// task.delDealByPe()
// task.delDealData()
task.exportThirdParty()
}
def delDealByPe(){
@@ -372,15 +374,29 @@ class DealPriceDataProcess {
return map
}
def exportThirdParty(){
def citys = CcbCommonUtils.citys_all
def results = []
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
for(def city : citys) {
def rets = findPeGteThirdParty3(city, 202305)
for(def map : rets){
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
}
println city +" "+ rets.size()
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方")
}
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202303
def month = 202304
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202304)
def rets = findPeGteThirdParty2(city, 202305)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -431,6 +447,28 @@ class DealPriceDataProcess {
return rets
}
//城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE
def findPeGteThirdParty3(cityName, yearmonth){
def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15"
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
map.cityid = r.urbn_id
map.city = r.urbn_nm
map.data_id = r.gujia_fsid
map.region = r.gujia_rgon_nm
map.comm_id = r.gujia_comm_id
map.district_name = r.gujia_comm_nm
map.type = r.gujia_bds_clc_tp
map.clc_price = r.gujia_bds_avprc
map.price = r.gujiabds_to_mkprc_mny
map.ppe = r.gujia_mns_rto
rets << map
});
return rets
}
def delDealData(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d ->
@@ -79,9 +79,9 @@ class ExportOracleAllDataTest {
// exportCommCaseDetailGj()//
// // ok文件
exportInOk()
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
// exportInOk()
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
// task.addFjwPrice()
task.addFjwPrice()
// task.addCcbPrice()//oracle
// task.addCcbPriceFromGuided()
@@ -42,7 +42,7 @@ class FjwPricePkCcbPrice {
// task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice()
task.addCcbPrice2()//按id查
// task.addCcbPrice2()//按id查
// task.addCcbPrice3()//按city,region,name查
// task.standRegion()
// task.addGuidPrice()
@@ -217,17 +217,19 @@ class FjwPricePkCcbPrice {
}
def addFjwPrice(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['C'])
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d->
if(!d['N']){
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd"))
if(dbo){
d['K'] = dbo.avg_price
}
d['N'] = dist.region
// def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd"))
// if(dbo){
// d['K'] = dbo.avg_price
// }
}
}
})
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -83,8 +83,8 @@ class GuiDingPriceBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
saveToPgSql(rows)
//更新静态数据表
//
// //更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -33,8 +33,8 @@ import java.util.regex.*;
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
@@ -1287,18 +1287,18 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
//// def region = RegionData.formatRegion(d['A'], d['B'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
//// if(dist){
//// d['E'] = dist.region
//// d['F'] = dist._id.toString()
//// }
//// }
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
def region = RegionData.formatRegion(d['B'], d['C'])
if(region){
def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
if(dist){
d['G'] = dist.region
d['H'] = dist._id.toString()
}
}
// if(!d['F']){
// def dist = CommonDatautils.findDistrictById(d['A'])
////
@@ -1308,30 +1308,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
// d['F'] = dist.status
// }
// }
//
// }
//},0)
def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
if(count && count==1){
def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
d['E'] = dist.city
}
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['J'] = dist.status
// }
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// d['F'] = newRegion
},0)
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
// d['E'] = dist.city
// }
//
//// def dist = CommonDatautils.findDistrictById(d['A'])
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['J'] = dist.status
//// }
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
//},0)
//ExcelUtils.modify(fileName, {d->
// if(d['A']){
// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'"
@@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
saveToPgSql(rows)
@@ -45,6 +45,7 @@ class TradeStatLog {
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
// task.deleteDistrict()
// task.deleteDealData()
// task.deleteDealData2()
@@ -92,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-02-01"
def date = "2023-03-01"
def endDate = "2023-04-01"
def newDate = "2023-05-01"
def startDate = "2023-03-01"
def date = "2023-04-01"
def endDate = "2023-05-01"
def newDate = "2023-06-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap)
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
@@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
citys = CcbCommonUtils.getRunningCitys(args)
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
@@ -0,0 +1,261 @@
package com.fangjia.script.chongqing
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
class FumingTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new FumingTest();
task.run();
}
def run() {
def path = "D:\\数据\\富民\\富民测试500-v7.xlsx"
// match(path)
// addPrice(path)
addThirdPrice(path)
// addLastPrice(path)
// addFacLevel(path)
}
def addFacLevel(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['K'] && row['I']){
// def obj = findCommunitySummary(row['C'])
// if(obj){
// row['K'] = obj.consumption_rating
// }else{
row['K'] = getNearByDistrictLevel(row['C'], row['I'])
// }
}
})
}
/**
* 一公里小区数量
*/
def getNearByDistrictLevel(def id, def price){
BasicDBObject district = DbUtils.seaweed.findById(id);
def level = null
if(district){
List<DBObject> list = DbUtils.findAroundDistrict(district);
println id +" 共:"+list.size()+" "+ price
if(list!=null&&list.size()>0){
for(def obj : list){
def summary = findCommunitySummary(obj._id.toString())
if(summary && summary.avg_price && summary.consumption_rating){
def pre = java.lang.Math.abs((summary.avg_price - price)*100/price)
println summary.name +" "+pre +" "+ summary.avg_price
if(pre <= 10){
level = summary.consumption_rating
break;
}
}
}
}
}
return level;
}
def addLastPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Z'] ){
if(row['W']){
row['Z'] = row['W']
}else if(row['R']){
row['Z'] = row['R']
}else if(row['I']){
row['Z'] = row['I']
}
if(row['Z']){
row['J'] = (int)(row['Z']*row['B'].toString().toDouble())
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['N'] ){
def city = row['C']
def address = row['A']
def region = row['D']
def name = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['N'] = jsonObject.get("id")
row['O'] = jsonObject.get("city")
row['P'] = jsonObject.get("district")
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("propertyType")
row['S'] = jsonObject.get("address")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['C']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['D'] = dist.city
row['E'] = dist.region
row['F'] = dist.name
row['G'] = dist.estate_type2
row['H'] = dist.address
// row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['I'] = priceObj.avg_price
row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble())
row['R'] = priceObj.avg_price_2b
}
}
}
})
}
def addThirdPrice(path){
// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连']
def allMap = [:]
// for(def c : citys){
// def cityMap = findThirdPartyPrice(c)
// allMap << cityMap
// }
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['L'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['M'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['N'] = priceMap.get("anjuke_esf_price")
}
}
def map = findAvgPrice(row['C'], cityCodeMap.get(row['D']))
if(map){
row['O'] = map.avg_price
row['P'] = map.credit_value
}
})
}
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
if(d['C']){
def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
if(map){
d['K'] = map.avg_price
d['L'] = map.calc_type
// d['AI'] = map.credit_value
d['M'] = map.month
}
}
})
}
def findThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def findCommunitySummary(id){
def obj = null
def sql = "select * from community_summary where id='${id}'"
ElasticUtils.executeSqlQuery(sql, {
obj = it
})
return obj
}
}
@@ -40,19 +40,25 @@ class HailanTest {
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new HailanTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\海南\\统计.xlsx"
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
// addThirdPrice(inputfile)
// match(inputfile)
// tradeAvgPrice(inputfile)
// caluateTotalPrice(inputfile)
// addPrice(inputfile)
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
// addHouse(inputfile)
// stat(inputfile)
hailanStat(inputfile)
// hailanStat(inputfile)
addFzFjwPrice(inputfile)
}
def addHouse(inputfile){
@@ -533,13 +539,13 @@ class HailanTest {
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){
String city = row['H']//RegionData.formatCity(row['B'])
if(row['A']){
String city = "福州"//RegionData.formatCity(row['B'])
if(city){
def address = row['C']
def address = row['A']
// def region = row['C']
println "[match] city:"+city+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E'])
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("name")
@@ -554,6 +560,8 @@ class HailanTest {
})
}
def stat(path){
ExcelUtils.modify(path, { row ->
if(row['L'] && !row['H']){
@@ -700,4 +708,60 @@ class HailanTest {
}
return prices;
}
def tradeAvgPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){
def dist = CommonDatautils.findDistrictById(row['I'])
if(dist){
def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01");
row['J'] = (int)tradeAvgPrice
}
}
})
}
def addThirdPrice(path){
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate)
def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95)
}
}
})
}
def caluateTotalPrice(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['J']){
row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble())
}else if(row['K']){
row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble())
}
})
}
def addFzFjwPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['B']) {
def q = new BasicDBObject()
q.put("city", "福州")
q.put("region", row['A'])
q.put("name", row['B'])
q.put("month", 202304)
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if (obj) {
row['D'] = obj.avg_price
}
}
})
}
}
@@ -26,7 +26,7 @@ class MingShenPocTest {
def run(){
def path = "D:\\数据\\民生\\poc测试.xlsx"
//9级标准化
//structureV2(path)//针对海南poc地址数据此接口不合适使用
structureV2(path)//针对海南poc地址数据此接口不合适使用
//省市区拆分
// ner(path)//城市,区域带市带区
// match(path)
@@ -34,7 +34,7 @@ class MingShenPocTest {
// addLatLng(path)
// addAroundPrice(path)
// addAuthor(path)
addHouse(path)
// addHouse(path)
}
def structureV2(path){
@@ -0,0 +1,197 @@
package com.fangjia.script.chongqing
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class RuiFenPocTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new RuiFenPocTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\瑞丰\\房产信息-地址V3-1.xlsx"
// match(inputfile)
// addValue(inputfile)
// addThirdPrice(inputfile)
lastPrice(inputfile)
}
def lastPrice(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(!row['X'] && row['S']){
def mod = java.lang.Math.abs((row['S'].toString().toDouble()-row['H'].toString().toDouble())*100/row['H'].toString().toDouble())
if(mod <= 10){
row['U'] = row['H']
}else{
row['U'] = row['S']
}
}else{
if(row['K']){
row['U'] = row['K']
}else{
row['U'] = row['H']
}
}
})
}
def addThirdPrice(path){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F'])
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['L'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['M'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['N'] = priceMap.get("anjuke_esf_price")
}
}
def map = findAvgPrice(row['I'], cityCodeMap.get(row['D']))
if(map){
row['O'] = map.avg_price
row['P'] = map.credit_value
}
})
}
def addValue(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
def dist = CommonDatautils.findDistrictById(row['I'])
if(dist){
row["D"] = dist.city
row["E"] = dist.region
row["F"] = dist.name
row["G"] = dist.address
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['H'] = priceObj.avg_price
row['K'] = priceObj.avg_price_2b
}
}
})
}
def match(path) {
ExcelUtils.multiThreadingModify(path, { row ->
def city = "绍兴"
if (row['B'] && !row["D"]) {
def address = row['B']
println "[match] city:" + city + " address:" + address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null)
if(!row["D"]){
if (jsonObject != null && jsonObject.precise != -1) {
row["D"] = jsonObject.city
row["E"] = jsonObject.district
row["F"] = jsonObject.name
row["G"] = jsonObject.address
row["I"] = jsonObject.id
def priceObj = CommonDatautils.findDistrictPrice(jsonObject.city, jsonObject.district, jsonObject.name)
if(priceObj){
row['H'] = priceObj.avg_price
}
}
}else {
if (jsonObject != null && jsonObject.precise != -1) {
if(row['J']){
def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id
row['K'] = str
}else{
def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id
row['J'] = str
}
}
}
}
})
}
def findThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
}
@@ -1,11 +1,14 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import com.fangjia.core.util.HttpRequest;
@@ -23,6 +26,12 @@ import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
class CommonDatautils {
@@ -73,6 +82,19 @@ class CommonDatautils {
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -230,6 +252,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "长滩美墅三期")
DBObject keys = new BasicDBObject();
if(fields){
@@ -781,7 +804,7 @@ class CommonDatautils {
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", new BasicDBObject('$in', regions))
@@ -1001,7 +1024,7 @@ class CommonDatautils {
q.put("cat", cat);
q.put("region", region)
q.put("name", name);
q.put("status", 0)
q.put("visible", 0)
DBObject keys = new BasicDBObject();
if(fields){
@@ -3418,11 +3441,27 @@ class CommonDatautils {
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def sqmetreRentMap = [:]
for(def d : dbcursor){
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
// sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price)
sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price)
}
return sqmetreRentMap;
}
static def findAllSqmetreRentPrices(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("cat","district")
q.put("weekend", weekend)
def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000)
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def prices = []
for(def d : dbcursor){
prices << d.avg_price
}
return prices;
}
static def findNewSqmetreRent(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
@@ -3490,5 +3529,373 @@ class CommonDatautils {
}
return obj
}
static def findDistrictAnalyStat(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q)
return analyData
}
static def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
.must(date_range).must(area_range)
if(cat){
queryBuilder.must(QueryBuilders.termQuery("cat", cat))
}
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
// println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
//// println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
// continue;
// }
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(seawater.height && seawater.height>60){
continue;
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("lianjia_esf_price", db.beike_esf_price)
}
}
return map
}
/**
* 根据经纬度获取区域
* @param city
* @param location
* @return
*/
static def getRegionByLocation(city, def location){
def json = GaoDeApiUtils.geocode(location);
def region = null;
if (json && json.info=="OK") {
def regeocode = json.regeocode
if(regeocode && regeocode.addressComponent){
region = regeocode.addressComponent.district
if(!region){
region = regeocode.addressComponent.township
}
}
}
if(region){
region = RegionData.formatRegion(city, region)
}
return region;
}
}
@@ -0,0 +1,59 @@
package com.fangjia.script.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.mongodb.BasicDBObject
class MasterNameInAlias {
static main(args) {
def task = new MasterNameInAlias()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def allList = []
for(def city : citys){
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
def cityList = []
cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"]
for(def dist1 : disticts){
def distList = []
for(def dist2 : disticts){
if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){
if(dist2.alias && dist2.alias.contains(dist1.name)){
def distance = ''
if(dist1.lng && dist2.lng){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d);
}
distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance]
dist2.alias.remove(dist1.name)
def d = new BasicDBObject()
d.put("_id", dist2._id)
def udata = new BasicDBObject()
udata.put("alias", dist2.alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
}
}
}
if(distList){
cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', '']
cityList.addAll(distList)
// cityList << []
}
}
if(cityList){
allList.addAll(cityList)
println city +" "+cityList.size()
}
}
// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据")
}
}
@@ -35,7 +35,7 @@ class FjtResponseTimeAnaly extends BaseBatch{
@Override
public void process(Object args) {
//run();
jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了?")
jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了2?")
}
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -82,20 +83,21 @@ class DistrictInfoExportBatch extends BaseBatch{
}
}
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
for(def dist : removeDists){
def key = dist.city +"##"+dist.region+"##"+dist.name
if(!distKeys.contains(key)){
distKeys << key
districts << dist
}
}
// for(def dist : removeDists){
// def key = dist.city +"##"+dist.region+"##"+dist.name
// if(!distKeys.contains(key)){
// distKeys << key
// districts << dist
// }
// }
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
saveOrUpdateFzDist(districts)
// saveOrUpdateFzDist(districts)
remarkRemoveDist(removeDists)
def fzDistricts = getFzDistricts(city)
for(def rMap : fzDistricts){
@@ -185,6 +187,15 @@ class DistrictInfoExportBatch extends BaseBatch{
}
def remarkRemoveDist(removeDists){
for(def rd : removeDists){
def udate = new BasicDBObject()
udate.put("status",1)
udate.put("m_date", new Date())
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
}
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
@@ -311,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def s = str.toString();
index++;
if(s && s!="null"){
s = s.replace('"','\\"')
// if(s.toString().contains(",")){
// s = csvHandlerStr(s.toString())
//
@@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch {
if(weekendPriceObj.value_type!=1){
if(dist.city == "¸£ÖÝ"){
if(Math.abs(mom)*100>3){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
if(dist.region != "Âíβ"){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
}else{
if(Math.abs(mom)*100>10){
@@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{
// sb.append(s+ "@|@")
// }
// }
sb.append(StringUtils.join(row.toArray(), "@|@"))
def str = StringUtils.join(row.toArray(), "@|@")
str = str.replace('"','\\"')
sb.append(str)
sb.append(System.getProperty("line.separator"));
println index
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
@@ -11,26 +12,91 @@ import com.mongodb.DBCollection
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def datas = FileUtils.readLines("D:\\Êý¾Ý\\ÆäËü\\FJW_COMM_20221015.csv")
def rets = []
for(def d : datas){
def arrs = d.split("@")
rets << ["commId":arrs[0], "avg_price":arrs[4]]
}
for(def r : rets){
if(r.avg_price){
def udata = new BasicDBObject()
udata.put("date", 202210)
udata.put("avg", r.avg_price.toInteger())
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
}else{
println r.commId +" ²»´æÔÚ"
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
def city = RegionData.formatCity("余姚")
d['D'] = city
if(city){
d['E'] = RegionData.formatRegion(city, d['C'])
}
// if(!d['D']){
// def citys = CommonDatautils.getCityByRegion(d['A'])
// if(citys && citys.size()==1){
// d['F'] = citys.get(0)
// }
// }
})
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
//def m = [:]
//for(def d : datas){
// m.put(d['B'], d)
//}
//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d->
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(dist){
// def ret = m.get(dist._id.toString())
// if(ret){
// d['E'] = ret['J']
// d['F'] = ret['N']
// }
// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS)
// if(dbo){
// if(dbo.beike_esf_price){
// d['G'] = dbo.beike_esf_price
// }else if(dbo.beike_esf_price){
// d['G'] = dbo.beike_esf_price
// }
//
// if(dbo.maitian_esf_price){
// d['H'] = dbo.maitian_esf_price
// }
// }
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd"))
// if(dbo){
// d['K'] = dbo.avg_price
// d['L'] = dbo.value_type
// }
//
// }
//})
def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", 202304)
return AVERAGE_STATISTICS.findOne(q)
}
//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
//def rets = []
//for(def d : datas){
// def arrs = d.split("@")
// rets << ["commId":arrs[0], "avg_price":arrs[4]]
//}
//
//for(def r : rets){
// if(r.avg_price){
// def udata = new BasicDBObject()
// udata.put("date", 202210)
// udata.put("avg", r.avg_price.toInteger())
// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
// }else{
// println r.commId +" 不存在"
// }
//}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -29,8 +29,11 @@ class AddAvgpriceBatch {
}
def test(){
//发重算均价命令
addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd'))
def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"])
for(def dist : districts){
//发重算均价命令
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
}
}
def run(){
@@ -96,17 +96,28 @@ class CityStatExport {
}
def regionStat(){
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州']
def ret = []
ret << ["城市","区域","小区总量"]
for(def city : citys){
def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){
def row = []
row << city
row << r
def districts = CommonDatautils.findDistricts(city, r, null)
println city +" "+ r +" "+ districts.size()
row << CommonDatautils.queryCityRegionPrice(city, r, null)
def obj = CommonDatautils.findRegionPriceRate(city, r)
row << obj.price_coverage*100
row << districts.size()
ret << row
}
}
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县")
}
@@ -272,6 +272,45 @@ class FiledStrand {
return null
}
static def processName(name){
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
if (name.contains('(') && name.contains(')')){
def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1)
name = name.replace(rep_str, '')
}
if (name.contains('(') && name.contains(')')){
def rep_str = name.subsring(name.index('('),name.rfind(')')+1)
name = name.replace(rep_str, '')
}
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
name = name.replace(rep_str, '')
}
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
name = name.replace(rep_str, '')
}
if (name.find('(')!=-1 && name.find(')')!=-1){
def rep_str = name[name.index('('):name.rfind(')')+1]
name = name.replace(rep_str, '')
}
if (name.find('「')!=-1 && name.find('」')!=-1){
def rep_str = name[name.index('「'):name.rfind('」')+1]
name = name.replace(rep_str, '')
}
return name
}
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
@@ -579,6 +618,12 @@ class FiledStrand {
}else{
return null;
}
}else if(estateType2=="写字楼"){
if(estateCharege>=0.35 && estateCharege<=60){
return estateCharege;
}else{
return null;
}
}
}catch (Exception e){
println "estateCharege ת²»ÁË "+estateCharege
@@ -567,7 +567,7 @@ class OpenApiUtils {
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3"
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
@@ -580,5 +580,79 @@ class OpenApiUtils {
}
//源地址的完整性进行验证
static def addrVerify(city, address){
def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
// 省、市、区、街道、社区居委、POI补全
static def complete(city, address){
def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
//五级(省/市/区/街道/社区居委)解析、补全
static def division(city, address){
def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
//识别地址中省、市、区、道路、POI同时进行地址验证
static def structure(city, address){
def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
/**
* 招商银行测试
* @param city
* @param address
* @return
*/
static def cmbTest(city, address){
def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
@@ -69,29 +70,43 @@ import com.mongodb.DBObject
//
//
//})
println "开始统计"
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1)
def map = [:]
for(def d : datas){
def prices = map.get(d['B'])
if(!prices){
prices = []
map.put(d['B'], prices)
}
prices << d['G']
}
def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
def rows = []
rows << ["区域","数量", "均价"]
regions.each{e->
def prices = map.get(e)
def price = (prices.sum()/prices.size()).toInteger()
rows << [e, prices.size(), price]
}
//map.each{region,prices->
// def price = (prices.sum()/prices.size()).toInteger()
// rows << [region, prices.size(), price]
//println "开始统计"
//def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1)
//def map = [:]
//for(def d : datas){
// def prices = map.get(d['B'])
// if(!prices){
// prices = []
// map.put(d['B'], prices)
// }
// prices << d['G']
//}
ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果")
//
//def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
//def rows = []
//rows << ["区域","数量", "均价"]
//
//regions.each{e->
// def prices = map.get(e)
// def price = (prices.sum()/prices.size()).toInteger()
// rows << [e, prices.size(), price]
//}
////map.each{region,prices->
//// def price = (prices.sum()/prices.size()).toInteger()
//// rows << [region, prices.size(), price]
////}
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果")
//ExcelUtils.multiThreadingModify("D:\\数据\\其它\\城市等级.xlsx",{d->
// d['C'] = RegionData.formatCity(d['A'])
//})
DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo);
ExcelUtils.modify("D:\\数据\\其它\\城市等级.xlsx",{d->
def q = new BasicDBObject()
q.put("display", d['C'])
def udata = new BasicDBObject()
udata.put("new_city_grade",d['E'].toString().toDouble().toInteger())
SERVER_AREA_CONN.update(q, new BasicDBObject('$set', udata))
})
@@ -0,0 +1,49 @@
package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
if(d['A'] && !d['E']){
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
println d['A'] +" "+d['C']
// def total = findDatas(d['A'], d['B'], d['C'])
// d['D'] = total
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
d['E'] = c
}
})
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name.contains(name) && d.type.contains("住宅")){
total ++
}
})
return total;
}
def findDatas(def city, def region, def name){
def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'"
def total = 0
ElasticUtils.executeSqlQuery(sql, { d->
total = d.count
})
return total;
}
//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
// def region = RegionData.formatCity(d['A'])
// if(!region && d['A']){
// println d['A']
// }
//})
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-04-01"
static def statDate = "2023-05-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -76,24 +76,24 @@ MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
}
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
//def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
//Calendar calendar = Calendar.getInstance()
//calendar.setTime(startDate)
//def count = 0
//def hashSet = []
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
//
//OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
// def result = JSONObject.fromObject(doc.res_map.result)
// def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
// data << item
// hashSet << result
//}
//
//
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
//ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
@@ -115,4 +115,27 @@ ExcelUtils.write("D:\\
//
//
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230524", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]]
OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode,
doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber]
data << item
hashSet << result
}
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
@@ -0,0 +1,207 @@
package com.fangjia.script.gj.data
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 按区域统计覆盖率
*/
class XiamenStat {
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
public static final def DISTRICT_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
]
static main(args) {
def task = new XiamenStat();
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d->
if(d['D'] && d['D']=="全市"){
def dbo = CommonDatautils.findCityInfo(d['C'].replace("市",""))
if(dbo){
def detail = dbo.district_detail
if(detail){
d['E'] = dbo.existing_district_count//数据量
d['F'] = detail.township*100// 乡镇街道
d['G'] = detail.address*100// 地址
d['H'] = detail.addrAlias*100// 地址别名
d['I'] = detail.name*100// 名称
d['J'] = detail.alias*100// 别名
d['K'] = detail.alias*100// 映射别名
d['L'] = detail.lat2*100// 经纬度(百度)
d['M'] = detail.lat*100// 经纬度(高德)
d['N'] = detail.estateType2*100// 物业类型
d['O'] = detail.buildType*100// 建筑类型
d['P'] = detail.streetCommunity*100// 街道社区
d['Q'] = detail.committee*100// 居委会
d['R'] = detail.businessQuarter*100// 商圈
d['S'] = detail.schools*100// 对口小学
d['T'] = detail.kindergartens*100// 对口幼儿园
d['U'] = detail.loudongCount*100// 楼栋总数
d['V'] = detail.householdCount*100// 户数
d['W'] = detail.estateCharge*100// 物业费
d['X'] = detail.carSiteCount*100// 停车位
d['Y'] = ""// 电梯
d['Z'] = detail.propertyLife*100// 产权年限
d['AA'] = detail.houseProperty*100// 产权性质
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
d['AC'] = detail.landProperty*100// 小区性质
d['AD'] = detail.finishBuildingDate*100// 竣工时间
d['AE'] = detail.openDate*100// 交房时间
d['AF'] = detail.allArea*100// 占地面积
d['AG'] = detail.buildArea*100// 建筑面积
d['AH'] = detail.cubagePercent*100// 容积率
d['AI'] = detail.virescencePercent*100// 绿化率
d['AJ'] = detail.mainType*100// 主力户型
d['AK'] = detail.mainTypeArea*100// 主力户型面积
d['AL'] = detail.developer*100// 开发商
d['AM'] = detail.estateCompany*100// 物业公司
d['AN'] = dbo.price_coverage*100// 小区当前均价
}
}
}else{
def fields = DISTRICT_FIELDS_MAP.keySet().asList()
def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields)
def detail = districtDetail(regionDistricts.size(), regionDistricts);
if(detail){
d['E'] = regionDistricts.size()//数据量
d['F'] = detail.township*100// 乡镇街道
d['G'] = detail.address*100// 地址
d['H'] = detail.addrAlias*100// 地址别名
d['I'] = detail.name*100// 名称
d['J'] = detail.alias*100// 别名
d['K'] = detail.alias*100// 映射别名
d['L'] = detail.lat2*100// 经纬度(百度)
d['M'] = detail.lat*100// 经纬度(高德)
d['N'] = detail.estateType2*100// 物业类型
d['O'] = detail.buildType*100// 建筑类型
d['P'] = detail.streetCommunity*100// 街道社区
d['Q'] = detail.committee*100// 居委会
d['R'] = detail.businessQuarter*100// 商圈
d['S'] = detail.schools*100// 对口小学
d['T'] = detail.kindergartens*100// 对口幼儿园
d['U'] = detail.loudongCount*100// 楼栋总数
d['V'] = detail.householdCount*100// 户数
d['W'] = detail.estateCharge*100// 物业费
d['X'] = detail.carSiteCount*100// 停车位
d['Y'] = ""// 电梯
d['Z'] = detail.propertyLife*100// 产权年限
d['AA'] = detail.houseProperty*100// 产权性质
d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分)
d['AC'] = detail.landProperty*100 // 小区性质
d['AD'] = detail.finishBuildingDate*100// 竣工时间
d['AE'] = detail.openDate*100// 交房时间
d['AF'] = detail.allArea*100// 占地面积
d['AG'] = detail.buildArea*100// 建筑面积
d['AH'] = detail.cubagePercent*100// 容积率
d['AI'] = detail.virescencePercent*100// 绿化率
d['AJ'] = detail.mainType*100// 主力户型
d['AK'] = detail.mainTypeArea*100// 主力户型面积
d['AL'] = detail.developer*100// 开发商
d['AM'] = detail.estateCompany*100// 物业公司
// d['AN'] = ""// 小区当前均价
}
}
})
}
def districtDetail(def disCount, def regionDistricts) {
def countMap = [:]
for(def dbo : regionDistricts) {
DISTRICT_FIELDS_MAP.each() {key,skey->
if (isValid(dbo.get(key))) {
if(countMap.containsKey(key)) {
countMap.put(key, countMap.get(key) + 1)
}else {
countMap.put(key, 1)
}
}
}
}
def districtDetail = [:];
DISTRICT_FIELDS_MAP.each() {key,skey->
if(countMap.containsKey(key)) {
districtDetail.put(skey, roundDouble(countMap.get(key), disCount))
}else {
districtDetail.put(skey, 0)
}
}
if(countMap.containsKey("lat") && countMap.containsKey("lng")) {
districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount));
}else {
districtDetail.put('latLng', 0);
}
if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) {
districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount));
}else {
districtDetail.put('latLng2', 0);
}
return districtDetail;
}
def CONTAINS_KWS = ["...", "暂无", "待定", "未知"]
def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."]
def isValid(def value) {
if (!value) {
return false;
}
if (value instanceof String) {
value = value.trim()
for(String ck : CONTAINS_KWS) {
if(value.contains(ck)) {
return false;
}
}
for(String ek : EQUALS_KWS) {
if(ek.equals(value)) {
return false;
}
}
} else if (value instanceof Number) {
if (value <= 0) {
return false;
}
} else if (value instanceof Collection || value instanceof Map) {
if (value.size() == 0) {
return false;
}
}
return true;
}
double roundDouble(int a, int base) {
if(a == 0 || base == 0) {
return 0
}
return ((double)a / (double)base * 10000.0).toInteger() / 10000.0
}
}
@@ -0,0 +1,117 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 通过房源更改电梯
*/
@BatchOwner(Watcher.GaoJuan)
class AddDistrictElevatorBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
static main(args) {
runBatch(new AddDistrictElevatorBatch(), args)
}
@Override
void process(Object args) {
run()
}
def run(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def dists = CommonDatautils.findAllDistricts(c, "district", ["city","region","name","alias", "build_type", "estate_type2", "is_elevator"])
addEvevatorProcess(c, dists)
}
}
def addEvevatorProcess(c, dists){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
println "小区量:"+dists.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<dists.size();i++){
final def dist = dists[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(!dist.is_elevator){
def houses = CommonDatautils.findHouses(dist.city, dist.region, null, 20200101, 20230701, dist.name, dist.alias, dist.estate_type2)
def haveHouses = []
for(def h : houses){
if(h.height){
haveHouses << h
}
}
def newHouses = analyElevator(haveHouses, dist.estate_type2)
if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size()
BasicDBObject udata = new BasicDBObject()
if(dist.estate_type2 == "别墅"){
udata.put("is_elevator", false)
}else{
udata.put("is_elevator", true)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
rets << dist
}else{
if(dist.build_type && dist.build_type.contains("高层")){
BasicDBObject udata = new BasicDBObject('is_elevator',true)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
rets << dist
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
println c +" "+ rets.size()
// return retMap
}
def analyElevator(houses, estateType){
def heights = []
for(def h : houses){
if(estateType=="别墅"){
if(h.height && h.height<=4) {
heights << h.height
}
}else{
if(h.height && h.height>=10){
heights << h.height
}
}
}
return heights
}
}
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 小区字段入库(自动入库)
* 小区字段填充入库(自动入库)
*/
@BatchOwner(Watcher.GaoJuan)
class DistrictFieldAddBatch extends BaseBatch{
@@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = ["漳州"]//CommonDatautils.findAllCity().reverse()
def citys = CommonDatautils.findAllCity().reverse()
def analysRets = []
@@ -95,7 +95,7 @@ class FangjiaClientUtils {
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -0,0 +1,79 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 写字楼字段清洗入库(自动入库)
*/
@BatchOwner(Watcher.GaoJuan)
class OfficeFieldCleanBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new OfficeFieldCleanBatch(), args)
}
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
for(def c : citys){
def offices = findAllOffice(c)
strandField(offices)
}
}
def strandField(offices){
def fields = ["property_fee","build_area","","","","","","","","","",""]
for(def office : offices){
fields.each{
def newVal = null
if(it == "property_fee"){
newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼")
newVal = newVal.toString()
}
if(it =="all_area" || it =="build_area"){
newVal = FiledStrand.processArea(office.get(it))
if(newVal && newVal > 5000){
newVal = newVal.toString()
}
}
if(it =="name"){
newVal = FiledStrand.processName(office.get(it))
if(newVal){
newVal = newVal.toString()
}
}
}
}
}
def findAllOffice(city){
def sql = "select * from office where city='${city}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
})
return rets
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -15,7 +16,48 @@ class SameDistName {
static main(args) {
def task = new SameDistName()
task.run()
// task.run()
//地址和地址别名完全一致
// task.addressPocess()
}
def addressPocess(){
def citys = CommonDatautils.findAllCity();
for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"])
if(dists){
def index = 0
for(def dist : dists){
if(dist.address){
// if(dist.address.toString().endsWith("弄")){
// dist.address = dist.address.toString().replace("弄","号")
// }
if(dist.addr_alias && dist.addr_alias.contains(dist.address)){
def new_address_alias = []
for(def addr : dist.addr_alias){
if(!new_address_alias.contains(addr) && addr!=dist.address){
new_address_alias << addr
}
}
index ++
println "[${index}] "+dist.city +" "+dist.name+" 旧:"+dist.addr_alias+" 新:"+ new_address_alias
def d = new BasicDBObject()
d.put("_id", dist._id)
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("addr_alias", new_address_alias)
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
}
}
}
}
}
}
def run(){
@@ -144,5 +144,8 @@ class SeaweedDataMatchProcess {
}
public static void main(def agrs){
doMatch("上海", "静安", "金家巷西弄", null)
}
}
@@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&
stat()
def stat(){
def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1)
def map=[:]
for(def d : datas){
def count = map.get(d['C'])
@@ -69,8 +69,8 @@ def findLogs(){
def findFangjiaLogs(){
def startDate = DateUtil.parseDate("20230329", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230429", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230531", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def list = []
@@ -84,14 +84,21 @@ def findFangjiaLogs(){
QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
// def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def q = QueryBuilders.fuzzyQuery("request", "/enquire")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
if (houses) {
for (def d : houses) {
def reqStr = d.request.toString()
if(!reqStr.contains("/enquire")){
continue
}
def mid = null
if (reqStr.contains("mid=")) {
@@ -107,5 +114,5 @@ def findFangjiaLogs(){
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志")
ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志")
}
@@ -0,0 +1,232 @@
package com.fangjia.script.gj.es
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils
import org.apache.lucene.queryparser.classic.QueryParser
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class HouseAnaly {
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new HouseAnaly()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def allRets = []
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"])
def rets = processing(districts)
println "[${city}] 写文件 "+rets.size()
allRets.addAll(rets)
}
ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析")
}
def processing(dists){
def rets = new CopyOnWriteArrayList()
rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(dist.alias && dist.alias.size()>=1){
def ret = analy(dist)
if(ret){
rets.addAll(ret)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def analy(dist){
def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是");
def alias = dist.alias
for(def aliasName : alias){
if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){
def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否");
if(aliaRet){
masterRet.addAll(aliaRet)
}
}
}
if(masterRet.size()>=2){
def masterAvgPrice = 0
def masterMidPrice = 0
def back = false
for(def row : masterRet){
if(row.get(4)=="是"){
masterAvgPrice = row.get(8)
masterMidPrice = row.get(9)
}else{
def pre1 = 0
def pre2 = 0
if(masterAvgPrice>0){
pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice)
}
if(masterMidPrice>0){
pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice)
}
if(pre1>=20 || pre2>=20){
back = true
}
}
}
if(back){
return masterRet
}
}
return null;
}
//小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量
def runAnalying(id, city, region, name, flag){
def houses = findHouse(city, region, name);
def rets = []
if(houses && houses.size()>=5){
def prices = []
for(def house : houses){
prices << house.avg_price
}
prices = prices.sort()
def average = (int)(prices.sum() / prices.size())
def midPrice = getMidPrice(prices)
def price10 = null
def price90 = null
if(houses.size()>=10){
def postion = Math.round(houses.size()/10)
price10 = prices.get((int)(postion-1))
price90 = prices.get((int)(houses.size()-postion-1))
}
def countMap = classType(houses)
// def pre = java.lang.Math.abs((midPrice-average)*100/average)
// if(pre>=20){
rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")]
// }
}
return rets
}
def classType(houses){
def district = 0
def villa = 0
def business = 0
for(def h : houses){
if(h.estate_type){
if(h.estate_type == "别墅"){
villa = villa + 1
continue
}else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){
business = business + 1
continue
}
}
def valla = isVilla(h.area, h.name, h.height, h.room)
if(valla){
villa = villa + 1
continue
}
district = district + 1
}
return ["district":district, "villa":villa, "business":business]
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getMidPrice(def prices){
def mid = prices.size()/2
if(mid > (int)mid){
return prices.get((int)mid)
}else{
def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1))
return (int)(total/2)
}
return null
}
def findHouse(def city, def region, def name){
def houses = []
try{
name = escapeExprSpecialWord(name)
String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'"
ElasticUtils.executeSqlQuery(sql,{ d->
def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type]
houses << r
// println r
})
}catch(Exception e){
e.printStackTrace()
}
return houses
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
@@ -21,13 +21,23 @@ class GaoDeMapSearchTest {
static main(args) {
def task = new GaoDeMapSearchTest();
def path = "D:\\数据\\光大\\光大测试-V2.xlsx"
def path = "D:\\数据\\富民\\富民测试500.xlsx"
// task.structureV2(path)
// task.match(path)
// task.mapSearch(path)
task.baiduMapSearch(path)
task.checkTrue(path)
// task.baiduMapSearch(path)
// task.checkTrue(path)
task.addFinancialLevel(path)
}
def addFinancialLevel(path){
ExcelUtils.multiThreadingModify(path, { row ->
def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A'])
if(results){
row['V'] = results.consumptionRating
}
});
}
def checkTrue(path){
@@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
//广州,佛山 重庆
static main(args) {
def task = new BeiKeCrawlByChrome();
@@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome {
println city +" 已运行"
continue
}
def newCity = RegionData.formatCity(city)
if(!newCity){
if(city=="昆山"){
newCity = "苏州"
}
println city +" 无法格式化"
}
def url = m[1]
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
@@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome {
def href = url+ r.attr("href")
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
if(city == "昆山"){
regionText = "昆山"
}
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println url +" 详情链接:"+detailUrls.size()
}
}
@@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, region)
def dbo = findTrade(durl , "贝壳找房")
if (!dbo) {
parse(city, e, region, driver)
}
}else{
stop = true;
break;
@@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome {
return detailUrls
}
def parse(city,e, region){
def parse(city,e, region, driver){
try {
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
if(title.contains("车位")){
detail.type = "车位"
}
detail.city = city
detail.district_name = arrs[0]
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
@@ -266,28 +289,48 @@ public class BeiKeCrawlByChrome {
}
detail.source = "贝壳找房"
if(detail.total_price && detail.avg_price){
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡",""))
detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","")
detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限",""))
if(detail.total_price && detail.avg_price){
detail.url = detailUrl
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
detail.floor_info = floorStr
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorInfo && floorInfo.isNumber()) {
detail.height = floorInfo.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}catch(Exception ex){
@@ -392,16 +435,16 @@ public class BeiKeCrawlByChrome {
}
def findTrade(def detail){
def findTrade(def url, source){
def q = new BasicDBObject()
// q.put("city", detail.city)
// q.put("region", detail.region)
// q.put("district_name", detail.district_name)
// q.put("area", detail.area)
// q.put("total_price", detail.total_price)
q.put("url", detail.url)
q.put("url", url)
// q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
q.put("source", source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
@@ -448,10 +491,13 @@ public class BeiKeCrawlByChrome {
btn.submit(); // 点击行为;
Thread.sleep(10000)
}catch(Exception e){
Thread.sleep(10000)
Thread.sleep(15000)
}
return driver
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.paimai
class DataFormatUtils {
static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000]
static def parseLift(def str){
if(str && str.contains("梯")){
def lift = str.toString().substring(0, str.toString().indexOf("梯"))
if(lift){
def val = chineseIntegerPart2Digit(lift)
if(val){
return (int)val
}
}
}
return null
}
static def parseHousehold(def str){
if(str && str.contains("梯") && str.contains("户")){
def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户"))
def val = chineseIntegerPart2Digit(household)
if(val){
return (int)val
}
}
return null
}
static def chineseIntegerPart2Digit(String chineseDigit)
{
Double finalDigit = 0.0;
Integer r = 1;
for(int i = chineseDigit.length()-1; i>=0;i--)
{
Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i)));
if( val >= 10 && i == 0 )
{
if (val > r)
{
r = val;
finalDigit = finalDigit +val;
}
else
{
r = r * val;
}
}
else if(val >= 10)
{
if (val > r)
{
r = val;
}
else
{
r = r * val;
}
}
else
{
finalDigit = finalDigit + r*val;
}
}
return finalDigit;
}
static String getValue(String s) {
if (s && s.contains("暂无")) {
return null
}
return s
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.ccb.FileUtils
@@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -196,9 +197,22 @@ public class LianjiaCrawlByChrome {
println city +" 已运行"
continue
}
def newCity = RegionData.formatCity(city)
if(!newCity){
if(city=="昆山"){
newCity = "苏州"
}
println city +" 无法格式化"
}
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
driver.get(newUrl)
try{
driver.get(newUrl)
}catch(Exception e){
println newUrl +" 链接有问题!"
continue
}
//def detailUrls = getPageUrls(driver, cityUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
@@ -208,9 +222,11 @@ public class LianjiaCrawlByChrome {
for(def r : regions){
def regionText = r.text()
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
if(city == "昆山"){
regionText = "昆山"
}
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
}
}
@@ -282,13 +298,21 @@ public class LianjiaCrawlByChrome {
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
if(dealDate){
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver)
}
}else{
stop = true;
break;
}
}else{
stop = true;
break;
println url +" 成交时间有误!"
}
}
if(page>=100){
stop = true;
@@ -302,85 +326,116 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e, regionText){
def parse(city,e, regionText, driver){
try {
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
def price = e.select(".totalPrice .number").text()
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
if(title.contains("车位")){
detail.type = "车位"
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
detail.city = city
detail.district_name = arrs[0]
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
}
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
if(!detail.region){
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
str = str.substring(0, str.indexOf("',"))
def locArrs = str.split(",")
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
if(gmap){
detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat)
}
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = (int)(price.toDouble() * 10000)
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
}
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
}
if(detail.total_price && detail.avg_price){
def dbo = findTrade(detail)
if (!dbo) {
if (price && price.isNumber()) {
detail.total_price = (int)(price.toDouble() * 10000)
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = detailUrl
def detailDiv = JQueryUtils.select(".introContent", detailDoc)
def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层",""))
if(detail.total_price && detail.avg_price){
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
detail.floor_info = floorStr
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr2 && floorStr2.isNumber()) {
detail.height = floorStr2.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}
}catch(Exception ex){
@@ -484,16 +539,16 @@ public class LianjiaCrawlByChrome {
}
def findTrade(def detail){
def findTrade(def url, source){
def q = new BasicDBObject()
// q.put("city", detail.city)
// q.put("region", detail.region)
// q.put("district_name", detail.district_name)
// q.put("area", detail.area)
// q.put("total_price", detail.total_price)
q.put("url", url)
// q.put("trade_date", detail.trade_date)
q.put("url", detail.url)
q.put("source", detail.source)
q.put("source", source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
@@ -530,12 +585,14 @@ public class LianjiaCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(10000)
Thread.sleep(15000)
}catch(Exception e){
Thread.sleep(10000)
}
@@ -0,0 +1,36 @@
package com.fangjia.script.gj.region
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 格式化其它网站区域
*/
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl)
def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1)
for(def d : datas){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("recognizeCity", d['F'])
q.put("recognizeRegion", d['A'])
def count = SERVER_AREA_COLL.count(q)
if(!count){
def obj = new BasicDBObject()
obj.put("city", d['A'])
obj.put("recognizeRegion", d['A'])
obj.put("recognizeCity", d['F'])
obj.put("createdDate", new Date())
SERVER_AREA_COLL.save(obj)
}else{
println d['A']+" "+ d['F'] +" 已存在"
}
}
@@ -58,8 +58,9 @@ class CZDataBatch extends BaseBatch{
def commids = findAllJnBankDatas(city)
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
districtRun(districts, statMap, commids);
FzDataUtils.removeNoUpdate(city, districts)
districtRun(districts, statMap, commids);
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
shops = mergeShop(shops);
@@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
if(dists){
for(def d : dists){
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
def distObj = null
if(d.comm_id){
distObj = CommonDatautils.findDistrictById(d.comm_id)
}else{
distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
}
if(distObj && distObj._id){
FzDataUtils.converDistrict(distObj, null, null);
if(d.type && d.type==1){
FzDataUtils.removeById(d.comm_id)
}else{
FzDataUtils.converDistrict(distObj, null, null);
}
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
updateTaskStatus(d.id, 2);
}
@@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("status", 5);//未处理
TASK_COLL.find(q).each {d->
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id]
}
return rets;
}
@@ -119,6 +119,13 @@ class FzDataUtils {
}
}
static def removeById(id){
def del_data = new BasicDBObject()
del_data.put("num_id", id)
DIST_COLL.remove(del_data)
}
static def saveDistrict(def m){
DBObject q = new BasicDBObject();
q.put("num_id",m.num_id)
@@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "역迦속潼±${city}×"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// if(city == "¸£ÖÝ"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
}
while(flag){
@@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(!dbo){
if(!dbo && isStandardRoad(copyObj.address)){
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
if(result.size()==1){
dbo = result[0];
@@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","¸£ÖÝ")
qdata.put("city","¸£ÖÝ")
// qdata.put("source","갛앙와")
// qdata.put("name",Pattern.compile("묠소酪.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
@@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
}
@Override
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
@@ -237,7 +237,7 @@ public class DbUtils{
def lng = district.lng / 10000000000.0
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0));
query.append('$sd', sdq)
def keys = new BasicDBObject()