This commit is contained in:
2025-03-18 17:00:31 +08:00
parent fc7ed5430c
commit ede742baaf
2 changed files with 574 additions and 0 deletions
@@ -0,0 +1,320 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.ExcelUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 写字楼字段填充入库(自动入库)
*/
@BatchOwner(Watcher.GaoJuan)
class OfficeFieldAddBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def sources = ["安居客app","办办网","房天下","中华写字楼", "点点租"]
//type:租售类别 wall:外墙
// def fieldsMap = ["elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","block":"block",
// "estate_type2":"property_type","estate_company":"property_company","estate_charge":"property_fee","all_area":"area",
// "standard_area":"standard_area","sight":"open_area","build_area":"build_area","build_type":"build_type","total_floor":"total_floor",
// "finish_building_date":"complete_time","loop_line_info":"ring_location","standard_height":"floor_height",
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","developer":"developer","fitment":"decoration","parking":"parking_cnt",
// "feature":"characteristic","description":"introduction","parking_fee":"car_fee","property_life":"property_life","bed_kind":"level",
// "lobby_height":"lobby_height","clear_height":"clear_height","foreign":"foreign","build_structure":"structure","address":"address",
// "lng":"location","open_date":"open_date","live_date":"live_date","real_use_percent":"room_rate","land_car_site":"land_car_site",
// "underground_car_site":"underground_car_site","land_floor":"land_floor","underground_floor":"underground_floor"]
def fieldsMap = ["build_area":"build_area"]
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","estate_charge":"property_fee","finish_building_date":"complete_time",
// "bed_kind":"level","land_car_site":"land_car_site","underground_car_site":"underground_car_site","parking_fee":"car_fee",
// "elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","clear_height":"clear_height",
// "foreign":"foreign","standard_area":"standard_area","foreign":"foreign","parking_fee":"car_fee","estate_company":"property_company","developer":"developer","all_area":"area",
static main(args) {
// runBatch(new OfficeFieldAddBatch(), args)
def task = new OfficeFieldAddBatch()
// task.fieldTempProcess()
task.deleteNotNormalOffice()
}
@Override
void process(Object args) {
def citys = ["深圳"]//CommonDatautils.findAllCity()
def fields = fieldsMap.keySet() as List
def analysRets = []
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
sources.each {source->
def esRets = findAllEsDataByCity(c, source)
def updateRets = processFieldsFromMatch(c , esRets, source)
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def analyRet = []
analyRet << source
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analysRets << analyRet
}
}
println analysRets.toString()
}
def processFieldsFromMatch(c,matchRets, source){
def fields = fieldsMap.keySet() as List
fields << "city"
def fieldValueMap = [:]
for(def esOffice : matchRets){
// if(esOffice.name!="东方建富时尚广场"){
// continue
// }
def office = CommonDatautils.findNormalDistrict(esOffice.city, esOffice.region, esOffice.name, "office", fields)
if(!office){
// println c+" "+esOffice.name+" 不存在"
continue
}
def newValues = strandField(c, office, esOffice, source)
if(newValues){
println office._id.toString()+" "+newValues.toString()
fieldValueMap.put(office._id.toString(), newValues)
}
}
return fieldValueMap;
}
// ["loop_line_info":"ring_location","feature":"characteristic","description":"introduction","lng":"location","open_date":"open_date"]
def strandField( c, office, esOffice, source){
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = office.get(seaweedfield)
if(!val){
def esVal = esOffice.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_type2"){
newVal = FiledStrand.processEstateType(esVal)
}
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), office.estate_type2)
if(newVal) {
newVal = newVal.toString()
}
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield == "loop_line_info"){
newVal = FiledStrand.processLoopLine(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area" || seaweedfield=="sight"){
newVal = FiledStrand.processArea(esVal.toString())
if(newVal){
newVal = newVal.toString()
}
}
if(seaweedfield =="car_site_count" || seaweedfield =="land_car_site" || seaweedfield =="underground_car_site"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, office.city)
}
if(seaweedfield =="block"){
newVal = FiledStrand.processBlock(office.city, office.region, esVal)
}
if(seaweedfield == "fitment"){
newVal = FiledStrand.processFitment(esVal)
}
if(seaweedfield == "build_type"){
newVal = FiledStrand.processBuildType(esVal, "office")
}
if(seaweedfield == "build_structure"){
newVal = FiledStrand.processBuildStructure(esVal)
}
if(seaweedfield == "bed_kind"){
newVal = FiledStrand.processLevel(esVal)
}
if(seaweedfield == "total_floor" || seaweedfield == "land_floor"){
newVal = FiledStrand.processFloor(esVal)
}
if(seaweedfield == "underground_floor"){
newVal = FiledStrand.processUndergroundFloor(esVal)
}
if(seaweedfield == "real_use_percent"){
newVal = FiledStrand.processRealUsePercent(esVal)
}
if(seaweedfield == "parking_fee"){
newVal = FiledStrand.processCarSiteRent(esVal)
println "[ ${source} ] old: "+esVal +" new:"+newVal
}
if(seaweedfield == "elevators_count"){
newVal = FiledStrand.processElevators(esVal)
}
if(seaweedfield == "person_elevators_count" && esVal.contains("客梯")){
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("客梯")))
}
if(seaweedfield == "goods_elevator_count" && esVal.contains("货梯")){
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("货梯")))
}
if(seaweedfield == "standard_height" || seaweedfield=="lobby_height" || seaweedfield=="clear_height"){
newVal = FiledStrand.processStandardHeight(esVal)
}
if(seaweedfield == "standard_area"){
newVal = FiledStrand.processStandardArea(esVal.toString())
}
if(seaweedfield == "finish_building_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "foreign"){
newVal = FiledStrand.processForeign(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
if(seaweedfield == "lng"){
try{
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
}
}catch(Exception e){
println esVal
}
}
}
}
}
return newValues;
}
def findAllEsDataByCity(city, source){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", source))
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "office2", queryBuilder);
}catch(Exception e){
datas = findAllEsDataByCity(city, source)
}
return datas;
}
def fieldUpdate(total,updateRets){
def countMap = [:]
for(def rm : updateRets){
def valMap = rm.value
fieldsMap.each{seaweedfield,esField->
if(valMap.containsKey(seaweedfield)) {
if(countMap.containsKey(seaweedfield)) {
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
}else{
countMap.put(seaweedfield, 1)
}
}
}
}
println countMap.toString()
def precentMap = [:]
fieldsMap.each() {key,skey->
if(countMap.containsKey(key)) {
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
}else {
precentMap.put(key, 0)
}
}
return precentMap;
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
def id = rm.key
def valMap = rm.value
BasicDBObject udata = new BasicDBObject()
valMap.each {k,v->
udata.put(k,v)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
}
}
def fieldTempProcess(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx",{d->
BasicDBObject udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("address", d['F'])
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
})
}
def deleteNotNormalOffice(){
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\需删除写字楼.xlsx",{d->
BasicDBObject udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("status", 1)
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
})
}
}
@@ -0,0 +1,254 @@
package com.fangjia.script.gj.es
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
class OfficeStat {
static main(args) {
def task = new OfficeStat()
// task.run()
// task.sampleFieldPrecent()
// task.cityRun()
task.tempStat()
}
def tempStat(){
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\写字楼租金统计.xlsx",{d->
def cityObj = CommonDatautils.findAlCityDetail(d['A'])
if(cityObj){
if(cityObj.new_city_grade==1000){
d['P'] = "一线"
}else if(cityObj.new_city_grade==1500){
d['P'] = "新一线"
}else if(cityObj.new_city_grade==2000){
d['P'] = "二线"
}else if(cityObj.new_city_grade==3000){
d['P'] = "三线"
}else if(cityObj.new_city_grade==4000){
d['P'] = "四线"
}else if(cityObj.new_city_grade==5000){
d['P'] = "五线"
}
}
// def count3s = []
// if(d['C']>=3){
// count3s << d['C']
// }
// if(d['D']>=3 ){
// count3s << d['D']
// }
// if(d['E']>=3 ){
// count3s << d['E']
// }
// if(d['F']>=3 ){
// count3s << d['F']
// }
// if(d['G']>=3 ){
// count3s << d['G']
// }
// if(d['H']>=3 ){
// count3s << d['H']
// }
// if(d['I']>=3 ){
// count3s << d['I']
// }
// if(d['J']>=3 ){
// count3s << d['J']
// }
// if(d['K']>=3 ){
// count3s << d['K']
// }
// if(d['L']>=3 ){
// count3s << d['L']
// }
// if(d['M']>=3 ){
// count3s << d['M']
// }
// if(d['N']>=3 ){
// count3s << d['N']
// }
// d['O'] = DataHandleUtils.getDoubleRound(4, count3s.size()/12)
},2)
}
def sampleFieldPrecent(){
def citys = CommonDatautils.findAllCity()
def rets = []
rets << ["城市","区域","名称","级别","面积","最高楼","装修","单价(元/平米/天)","发布时间","总量"]
citys.each { c ->
def regionMap = CommonDatautils.getRegionsByCity(c)
def ret = sampleFieldStat(c, regionMap, "office_rent")
rets.addAll(ret)
}
///home/goojia/gj
ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼租金样本字段覆盖率")
}
def run(){
def citys = CommonDatautils.findAllCity()
def rets = []
rets << ["城市","区域","近1个月样本量","近2个月样本量","近3个月样本量","近4个月样本量","近5个月样本量","近6个月样本量","近7个月样本量","近8个月样本量","近9个月样本量","近10个月样本量","近11个月样本量","近12个月样本量","样本覆盖率(区域走势)" ]
citys.each{c->
def regionMap = CommonDatautils.getRegionsByCity(c)
def ret = statHouse(c, regionMap, "office_rent")
rets.addAll(ret)
}
ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼区域租金样本覆盖率")
}
def cityRun(){
def citys = CommonDatautils.findAllCity()
def rets = []
rets << ["城市","区域","近1个月样本量","近2个月样本量","近3个月样本量","近4个月样本量","近5个月样本量","近6个月样本量","近7个月样本量","近8个月样本量","近9个月样本量","近10个月样本量","近11个月样本量","近12个月样本量","样本覆盖率(区域走势)" ]
citys.each{c->
def ret = statCityHouse(c, "office_rent")
rets.addAll(ret)
}
ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼城市租金样本覆盖率")
}
def statHouse(city, regionMap, cat){
def regions = regionMap.keySet()
def rows = []
def months = [20250201, 20250101,20241201,20241101,20241001,20240901,20240801,20240701,20240601,20240501,20240401,20240301,20240201]
regions.each{region->
def row = []
row << city
row << region
def count3s = []
for(def i=1;i<months.size();i++){
def count = findHouseCountByCat(city, region, cat, months[i], months[0])
row << count
if(count>=3){
count3s << count
}
}
def precent = DataHandleUtils.getDoubleRound(4, count3s.size()/12)
row << precent
println row
rows << row
}
return rows
}
def statCityHouse(city, cat){
def months = [20250201, 20250101,20241201,20241101,20241001,20240901,20240801,20240701,20240601,20240501,20240401,20240301,20240201]
def row = []
row << city
def count3s = []
for(def i=1;i<months.size();i++){
def count = findHouseCountByCat(city, null, cat, months[i], months[i-1])
row << count
if(count>=3){
count3s << count
}
}
def precent = DataHandleUtils.getDoubleRound(4, count3s.size()/12)
row << precent
println row
return row
}
def sampleFieldStat(city, regionMap, cat){
def regions = regionMap.keySet()
def rows = []
regions.each{region->
def results = findHouseByCat(city, region, cat, 20240201,20250201)
def row = []
def total = results.size()
def nameCount = 0
def levelCount = 0
def heightCount = 0
def areaCount = 0
def priceCount = 0
def dateCount = 0
def fitmentCount = 0
row << city
row << region
results.each{m->
def name = m.get("name")
if(name){
nameCount++
}
def level = m.get("level")
if(level){
levelCount++
}
def height = m.get("height")
if(height){
heightCount++
}
def fitment = m.get("fitment")
if(fitment){
fitmentCount++
}
def area = m.get("area")
if(area){
areaCount++
}
def day_rent_price = m.get("day_rent_price")
if(day_rent_price){
priceCount++
}
def p_date = m.get("p_date")
if(p_date){
dateCount++
}
}
if(total){
row << DataHandleUtils.getDoubleRound(4, nameCount/total)
row << DataHandleUtils.getDoubleRound(4, levelCount/total)
row << DataHandleUtils.getDoubleRound(4, areaCount/total)
row << DataHandleUtils.getDoubleRound(4, heightCount/total)
row << DataHandleUtils.getDoubleRound(4, fitmentCount/total)
row << DataHandleUtils.getDoubleRound(4, priceCount/total)
row << DataHandleUtils.getDoubleRound(4, dateCount/total)
}else{
row << 0
row << 0
row << 0
row << 0
row << 0
row << 0
row << 0
}
row << total
rows << row
println row
}
return rows
}
def findHouseCountByCat(def city, def region, def cat, def startDate, def endDate){
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
if(region){
sql = sql + " and region='${region}'"
}
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}
def findHouseByCat(def city, def region, def cat, def startDate, def endDate){
String sql = "select * from latest_seawater where cat='${cat}' and city='${city}' and region='${region}' and p_date >= ${startDate} and p_date<${endDate} "
def results =[]
ElasticUtils.executeSqlQuery(sql,{ data->
results << ["level":data.level,"name":data.district_name,"day_rent_price":data.total_price,"height":data.height,"area":data.area,"p_date":data.p_date,"fitment":data.fitment]
})
return results
}
}