From ede742baaf60f77af48bcd980182e06ec406fb04 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 18 Mar 2025 17:00:31 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/district/OfficeFieldAddBatch.groovy | 320 ++++++++++++++++++ .../fangjia/script/gj/es/OfficeStat.groovy | 254 ++++++++++++++ 2 files changed, 574 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldAddBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/OfficeStat.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldAddBatch.groovy new file mode 100644 index 000000000..741193c36 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldAddBatch.groovy @@ -0,0 +1,320 @@ +package com.fangjia.script.gj.district + +import com.fangjia.core.util.ExcelUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 写字楼字段填充入库(自动入库) + */ +@BatchOwner(Watcher.GaoJuan) +class OfficeFieldAddBatch extends BaseBatch{ + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + def sources = ["安居客app","办办网","房天下","中华写字楼", "点点租"] + + //type:租售类别 wall:外墙 +// def fieldsMap = ["elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","block":"block", +// "estate_type2":"property_type","estate_company":"property_company","estate_charge":"property_fee","all_area":"area", +// "standard_area":"standard_area","sight":"open_area","build_area":"build_area","build_type":"build_type","total_floor":"total_floor", +// "finish_building_date":"complete_time","loop_line_info":"ring_location","standard_height":"floor_height", +// "virescence_percent":"green_rate","cubage_percent":"volume_rate","developer":"developer","fitment":"decoration","parking":"parking_cnt", +// "feature":"characteristic","description":"introduction","parking_fee":"car_fee","property_life":"property_life","bed_kind":"level", +// "lobby_height":"lobby_height","clear_height":"clear_height","foreign":"foreign","build_structure":"structure","address":"address", +// "lng":"location","open_date":"open_date","live_date":"live_date","real_use_percent":"room_rate","land_car_site":"land_car_site", +// "underground_car_site":"underground_car_site","land_floor":"land_floor","underground_floor":"underground_floor"] + + def fieldsMap = ["build_area":"build_area"] + +// "virescence_percent":"green_rate","cubage_percent":"volume_rate","estate_charge":"property_fee","finish_building_date":"complete_time", +// "bed_kind":"level","land_car_site":"land_car_site","underground_car_site":"underground_car_site","parking_fee":"car_fee", +// "elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","clear_height":"clear_height", +// "foreign":"foreign","standard_area":"standard_area","foreign":"foreign","parking_fee":"car_fee","estate_company":"property_company","developer":"developer","all_area":"area", + + static main(args) { +// runBatch(new OfficeFieldAddBatch(), args) + + def task = new OfficeFieldAddBatch() +// task.fieldTempProcess() + task.deleteNotNormalOffice() + } + + @Override + void process(Object args) { + def citys = ["深圳"]//CommonDatautils.findAllCity() + def fields = fieldsMap.keySet() as List + def analysRets = [] + for(def c : citys){ + def totalCount = CommonDatautils.findAllDistrictCount("office", c) + + sources.each {source-> + def esRets = findAllEsDataByCity(c, source) + def updateRets = processFieldsFromMatch(c , esRets, source) + + def precentMap = fieldUpdate(totalCount, updateRets) + println c +" "+precentMap.toString() + updateDB(c, updateRets) + println "------------------------------------------------------------------------------" + + def analyRet = [] + analyRet << source + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analysRets << analyRet + } + } + println analysRets.toString() + } + + def processFieldsFromMatch(c,matchRets, source){ + def fields = fieldsMap.keySet() as List + fields << "city" + def fieldValueMap = [:] + for(def esOffice : matchRets){ +// if(esOffice.name!="东方建富时尚广场"){ +// continue +// } + def office = CommonDatautils.findNormalDistrict(esOffice.city, esOffice.region, esOffice.name, "office", fields) + if(!office){ +// println c+" "+esOffice.name+" 不存在" + continue + } + def newValues = strandField(c, office, esOffice, source) + if(newValues){ + println office._id.toString()+" "+newValues.toString() + fieldValueMap.put(office._id.toString(), newValues) + } + } + + return fieldValueMap; + } + +// ["loop_line_info":"ring_location","feature":"characteristic","description":"introduction","lng":"location","open_date":"open_date"] + def strandField( c, office, esOffice, source){ + def newValues = [:] + fieldsMap.each{seaweedfield,esField-> + def val = office.get(seaweedfield) + if(!val){ + def esVal = esOffice.get(esField) + if(esVal){ + def newVal = null + + if(seaweedfield == "estate_type2"){ + newVal = FiledStrand.processEstateType(esVal) + } + if(seaweedfield == "estate_charge"){ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), office.estate_type2) + if(newVal) { + newVal = newVal.toString() + } + } + if(seaweedfield == "build_date" || seaweedfield == "live_date"){ + newVal = FiledStrand.processFinishBuildingDate(esVal) + } + if(seaweedfield == "estate_company" || seaweedfield == "developer"){ + newVal = FiledStrand.processCompany(esVal) + } + if(seaweedfield =="virescence_percent"){ + newVal = FiledStrand.processVirescencePercent(esVal) + } + + if(seaweedfield == "loop_line_info"){ + newVal = FiledStrand.processLoopLine(esVal) + } + if(seaweedfield =="cubage_percent"){ + newVal = FiledStrand.processCubagePercent(esVal) + } + + if(seaweedfield =="all_area" || seaweedfield =="build_area" || seaweedfield=="sight"){ + newVal = FiledStrand.processArea(esVal.toString()) + if(newVal){ + newVal = newVal.toString() + } + } + + if(seaweedfield =="car_site_count" || seaweedfield =="land_car_site" || seaweedfield =="underground_car_site"){ + newVal = FiledStrand.processCarSite(esVal) + } + + if(seaweedfield =="property_life"){ + newVal = FiledStrand.processPropertyLife(esVal) + } + + if(seaweedfield =="address"){ + newVal = FiledStrand.processAddress(esVal, office.city) + } + if(seaweedfield =="block"){ + newVal = FiledStrand.processBlock(office.city, office.region, esVal) + } + + if(seaweedfield == "fitment"){ + newVal = FiledStrand.processFitment(esVal) + } + + if(seaweedfield == "build_type"){ + newVal = FiledStrand.processBuildType(esVal, "office") + } + if(seaweedfield == "build_structure"){ + newVal = FiledStrand.processBuildStructure(esVal) + } + if(seaweedfield == "bed_kind"){ + newVal = FiledStrand.processLevel(esVal) + } + + if(seaweedfield == "total_floor" || seaweedfield == "land_floor"){ + newVal = FiledStrand.processFloor(esVal) + } + + if(seaweedfield == "underground_floor"){ + newVal = FiledStrand.processUndergroundFloor(esVal) + } + + if(seaweedfield == "real_use_percent"){ + newVal = FiledStrand.processRealUsePercent(esVal) + } + if(seaweedfield == "parking_fee"){ + newVal = FiledStrand.processCarSiteRent(esVal) + println "[ ${source} ] old: "+esVal +" new:"+newVal + } + + if(seaweedfield == "elevators_count"){ + newVal = FiledStrand.processElevators(esVal) + } + if(seaweedfield == "person_elevators_count" && esVal.contains("客梯")){ + newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("客梯"))) + } + + if(seaweedfield == "goods_elevator_count" && esVal.contains("货梯")){ + newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("货梯"))) + } + + if(seaweedfield == "standard_height" || seaweedfield=="lobby_height" || seaweedfield=="clear_height"){ + newVal = FiledStrand.processStandardHeight(esVal) + } + + if(seaweedfield == "standard_area"){ + newVal = FiledStrand.processStandardArea(esVal.toString()) + } + + if(seaweedfield == "finish_building_date"){ + newVal = FiledStrand.processFinishBuildingDate(esVal) + } + if(seaweedfield == "foreign"){ + newVal = FiledStrand.processForeign(esVal) + } + + if(newVal){ + newValues.put(seaweedfield, newVal) + } + if(seaweedfield == "lng"){ + try{ + def locationMap = FiledStrand.processLngAndLat(esVal) + if(locationMap){ + newValues.put("lat", locationMap.lat) + newValues.put("lng", locationMap.lng) + newValues.put("lat2", locationMap.lat2) + newValues.put("lng2", locationMap.lng2) + } + }catch(Exception e){ + println esVal + } + } + } + } + } + return newValues; + } + + + def findAllEsDataByCity(city, source){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", source)) + def datas = null + try{ + datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "office2", queryBuilder); + }catch(Exception e){ + datas = findAllEsDataByCity(city, source) + } + + return datas; + } + + def fieldUpdate(total,updateRets){ + + def countMap = [:] + + for(def rm : updateRets){ + def valMap = rm.value + fieldsMap.each{seaweedfield,esField-> + if(valMap.containsKey(seaweedfield)) { + if(countMap.containsKey(seaweedfield)) { + countMap.put(seaweedfield, countMap.get(seaweedfield) + 1) + }else{ + countMap.put(seaweedfield, 1) + } + } + } + } + + println countMap.toString() + def precentMap = [:] + fieldsMap.each() {key,skey-> + if(countMap.containsKey(key)) { + precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) + }else { + precentMap.put(key, 0) + } + } + return precentMap; + } + + def updateDB(c, updateRets){ + + for(def rm : updateRets) { + def id = rm.key + def valMap = rm.value + + BasicDBObject udata = new BasicDBObject() + valMap.each {k,v-> + udata.put(k,v) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata)) + } + } + + + def fieldTempProcess(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx",{d-> + BasicDBObject udata = new BasicDBObject() + udata.put("m_date", new Date()) + udata.put("address", d['F']) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata)) + }) + } + + def deleteNotNormalOffice(){ + ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\需删除写字楼.xlsx",{d-> + BasicDBObject udata = new BasicDBObject() + udata.put("m_date", new Date()) + udata.put("status", 1) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata)) + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/OfficeStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/OfficeStat.groovy new file mode 100644 index 000000000..eaae19e43 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/OfficeStat.groovy @@ -0,0 +1,254 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +class OfficeStat { + static main(args) { + def task = new OfficeStat() +// task.run() +// task.sampleFieldPrecent() +// task.cityRun() + task.tempStat() + } + + def tempStat(){ + ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\写字楼租金统计.xlsx",{d-> + def cityObj = CommonDatautils.findAlCityDetail(d['A']) + if(cityObj){ + if(cityObj.new_city_grade==1000){ + d['P'] = "一线" + }else if(cityObj.new_city_grade==1500){ + d['P'] = "新一线" + }else if(cityObj.new_city_grade==2000){ + d['P'] = "二线" + }else if(cityObj.new_city_grade==3000){ + d['P'] = "三线" + }else if(cityObj.new_city_grade==4000){ + d['P'] = "四线" + }else if(cityObj.new_city_grade==5000){ + d['P'] = "五线" + } + } +// def count3s = [] +// if(d['C']>=3){ +// count3s << d['C'] +// } +// if(d['D']>=3 ){ +// count3s << d['D'] +// } +// if(d['E']>=3 ){ +// count3s << d['E'] +// } +// if(d['F']>=3 ){ +// count3s << d['F'] +// } +// if(d['G']>=3 ){ +// count3s << d['G'] +// } +// if(d['H']>=3 ){ +// count3s << d['H'] +// } +// if(d['I']>=3 ){ +// count3s << d['I'] +// } +// if(d['J']>=3 ){ +// count3s << d['J'] +// } +// if(d['K']>=3 ){ +// count3s << d['K'] +// } +// if(d['L']>=3 ){ +// count3s << d['L'] +// } +// if(d['M']>=3 ){ +// count3s << d['M'] +// } +// if(d['N']>=3 ){ +// count3s << d['N'] +// } +// d['O'] = DataHandleUtils.getDoubleRound(4, count3s.size()/12) + },2) + } + + def sampleFieldPrecent(){ + def citys = CommonDatautils.findAllCity() + def rets = [] + rets << ["城市","区域","名称","级别","面积","最高楼","装修","单价(元/平米/天)","发布时间","总量"] + citys.each { c -> + def regionMap = CommonDatautils.getRegionsByCity(c) + def ret = sampleFieldStat(c, regionMap, "office_rent") + rets.addAll(ret) + } + ///home/goojia/gj + ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼租金样本字段覆盖率") + } + def run(){ + def citys = CommonDatautils.findAllCity() + def rets = [] + rets << ["城市","区域","近1个月样本量","近2个月样本量","近3个月样本量","近4个月样本量","近5个月样本量","近6个月样本量","近7个月样本量","近8个月样本量","近9个月样本量","近10个月样本量","近11个月样本量","近12个月样本量","样本覆盖率(区域走势)" ] + citys.each{c-> + def regionMap = CommonDatautils.getRegionsByCity(c) + def ret = statHouse(c, regionMap, "office_rent") + rets.addAll(ret) + } + ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼区域租金样本覆盖率") + } + + def cityRun(){ + def citys = CommonDatautils.findAllCity() + def rets = [] + rets << ["城市","区域","近1个月样本量","近2个月样本量","近3个月样本量","近4个月样本量","近5个月样本量","近6个月样本量","近7个月样本量","近8个月样本量","近9个月样本量","近10个月样本量","近11个月样本量","近12个月样本量","样本覆盖率(区域走势)" ] + citys.each{c-> + def ret = statCityHouse(c, "office_rent") + rets.addAll(ret) + } + ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼城市租金样本覆盖率") + } + + def statHouse(city, regionMap, cat){ + def regions = regionMap.keySet() + def rows = [] + def months = [20250201, 20250101,20241201,20241101,20241001,20240901,20240801,20240701,20240601,20240501,20240401,20240301,20240201] + regions.each{region-> + def row = [] + row << city + row << region + + def count3s = [] + for(def i=1;i=3){ + count3s << count + } + } + def precent = DataHandleUtils.getDoubleRound(4, count3s.size()/12) + row << precent + println row + rows << row + } + return rows + } + + def statCityHouse(city, cat){ + def months = [20250201, 20250101,20241201,20241101,20241001,20240901,20240801,20240701,20240601,20240501,20240401,20240301,20240201] + def row = [] + row << city + + def count3s = [] + for(def i=1;i=3){ + count3s << count + } + } + def precent = DataHandleUtils.getDoubleRound(4, count3s.size()/12) + row << precent + println row + return row + } + + def sampleFieldStat(city, regionMap, cat){ + def regions = regionMap.keySet() + def rows = [] + regions.each{region-> + def results = findHouseByCat(city, region, cat, 20240201,20250201) + def row = [] + def total = results.size() + def nameCount = 0 + def levelCount = 0 + def heightCount = 0 + def areaCount = 0 + def priceCount = 0 + def dateCount = 0 + def fitmentCount = 0 + row << city + row << region + results.each{m-> + def name = m.get("name") + if(name){ + nameCount++ + } + + def level = m.get("level") + if(level){ + levelCount++ + } + + def height = m.get("height") + if(height){ + heightCount++ + } + + def fitment = m.get("fitment") + if(fitment){ + fitmentCount++ + } + + def area = m.get("area") + if(area){ + areaCount++ + } + def day_rent_price = m.get("day_rent_price") + if(day_rent_price){ + priceCount++ + } + + def p_date = m.get("p_date") + if(p_date){ + dateCount++ + } + } + if(total){ + row << DataHandleUtils.getDoubleRound(4, nameCount/total) + row << DataHandleUtils.getDoubleRound(4, levelCount/total) + row << DataHandleUtils.getDoubleRound(4, areaCount/total) + row << DataHandleUtils.getDoubleRound(4, heightCount/total) + row << DataHandleUtils.getDoubleRound(4, fitmentCount/total) + row << DataHandleUtils.getDoubleRound(4, priceCount/total) + row << DataHandleUtils.getDoubleRound(4, dateCount/total) + }else{ + row << 0 + row << 0 + row << 0 + row << 0 + row << 0 + row << 0 + row << 0 + } + + row << total + rows << row + println row + } + return rows + } + + + def findHouseCountByCat(def city, def region, def cat, def startDate, def endDate){ + + String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} " + if(region){ + sql = sql + " and region='${region}'" + } + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + return total + } + + def findHouseByCat(def city, def region, def cat, def startDate, def endDate){ + + String sql = "select * from latest_seawater where cat='${cat}' and city='${city}' and region='${region}' and p_date >= ${startDate} and p_date<${endDate} " + def results =[] + ElasticUtils.executeSqlQuery(sql,{ data-> + results << ["level":data.level,"name":data.district_name,"day_rent_price":data.total_price,"height":data.height,"area":data.area,"p_date":data.p_date,"fitment":data.fitment] + }) + return results + } +}