Merge branch 'script_gj' into 'master'
Script gj See merge request !1089
This commit is contained in:
+10
-10
@@ -61,25 +61,25 @@ class CcbCommonUtils {
|
||||
def countStr = args[0]
|
||||
if(countStr=="344"){
|
||||
citys = CcbCommonUtils.citys_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="92"){
|
||||
citys = CcbCommonUtils.citys_92
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="252"){
|
||||
citys = CcbCommonUtils.citys_252
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="all"){
|
||||
citys = CcbCommonUtils.citys_fjw_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}
|
||||
}else{
|
||||
citys = CcbCommonUtils.citys_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}
|
||||
println "运行城市数量:"+citys.size()
|
||||
return citys;
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class VillaHouseBatch extends BaseBatch{
|
||||
|
||||
def pgsqlInstance236 = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new VillaHouseBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city", "region", "name", "estate_type2"], "all")
|
||||
def villaDists = findVillaDists(dists)
|
||||
addVillaTypeFlag(villaDists)
|
||||
println "运行完成"
|
||||
}
|
||||
|
||||
def findVillaDists(dists){
|
||||
def villaDists = []
|
||||
for(def d : dists){
|
||||
if(d.estate_type2 == "别墅"){
|
||||
villaDists << d
|
||||
}
|
||||
}
|
||||
return villaDists
|
||||
}
|
||||
|
||||
def addVillaTypeFlag(def villaDists){
|
||||
for(def i=0;i<villaDists.size();i++){
|
||||
def dist = villaDists[i]
|
||||
|
||||
def sellIds = findHouseIdByCommId(dist._id.toString(), "310000")
|
||||
println dist._id.toString() +" "+sellIds.size()
|
||||
if(sellIds){
|
||||
def retMap = run(sellIds)
|
||||
if(retMap){
|
||||
println "最终修改:"+retMap.size()
|
||||
updateHouseFlag(retMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(sellIds){
|
||||
def retMap = new ConcurrentHashMap()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(sellIds.size())
|
||||
for(def i=0;i<sellIds.size();i++){
|
||||
final def houseId = sellIds[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
retMap = findHouseInfo(houseId, retMap)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findHouseIdByCommId(commid, cityId){
|
||||
def sql = "select gujia_sell_id from base.bigdata_forsale where urbn_id='${cityId}' and gujia_comm_id='${commid}'";
|
||||
def sellIds = []
|
||||
sqlInstance235.eachRow(sql,{r->
|
||||
sellIds << r.gujia_sell_id
|
||||
})
|
||||
return sellIds
|
||||
}
|
||||
|
||||
def findHouseInfo(sell_id, houseMap){
|
||||
def flag = findHouseDetail(sell_id)
|
||||
if(flag){
|
||||
houseMap.put(sell_id, flag)
|
||||
}
|
||||
return houseMap
|
||||
}
|
||||
|
||||
def findHouseDetail(houseId){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", houseId))
|
||||
def flag = null
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(5);
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def house = houses[k]
|
||||
def title = house.name
|
||||
println title
|
||||
if(title){
|
||||
if(title.contains("独栋")){
|
||||
flag = "独栋"
|
||||
}else if(title.contains("联排")){
|
||||
flag = "联排"
|
||||
}else if(title.contains("叠拼")){
|
||||
flag = "叠拼"
|
||||
}else if(title.contains("花园洋房")){
|
||||
flag = "花园洋房"
|
||||
}else if(title.contains("双拼")){
|
||||
flag = "双拼"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
def updateHouseFlag(retMap){
|
||||
retMap.each{sell_id,type->
|
||||
def sql = "update base.bigdata_forsale set villa_type='${type}' where gujia_sell_id='${sell_id}'"
|
||||
sqlInstance235.executeUpdate(sql)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 写字楼字段填充入库(自动入库)
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class OfficeFieldAddBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def sources = ["安居客app","办办网","房天下","中华写字楼", "点点租"]
|
||||
|
||||
//type:租售类别 wall:外墙
|
||||
// def fieldsMap = ["elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","block":"block",
|
||||
// "estate_type2":"property_type","estate_company":"property_company","estate_charge":"property_fee","all_area":"area",
|
||||
// "standard_area":"standard_area","sight":"open_area","build_area":"build_area","build_type":"build_type","total_floor":"total_floor",
|
||||
// "finish_building_date":"complete_time","loop_line_info":"ring_location","standard_height":"floor_height",
|
||||
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","developer":"developer","fitment":"decoration","parking":"parking_cnt",
|
||||
// "feature":"characteristic","description":"introduction","parking_fee":"car_fee","property_life":"property_life","bed_kind":"level",
|
||||
// "lobby_height":"lobby_height","clear_height":"clear_height","foreign":"foreign","build_structure":"structure","address":"address",
|
||||
// "lng":"location","open_date":"open_date","live_date":"live_date","real_use_percent":"room_rate","land_car_site":"land_car_site",
|
||||
// "underground_car_site":"underground_car_site","land_floor":"land_floor","underground_floor":"underground_floor"]
|
||||
|
||||
def fieldsMap = ["build_area":"build_area"]
|
||||
|
||||
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","estate_charge":"property_fee","finish_building_date":"complete_time",
|
||||
// "bed_kind":"level","land_car_site":"land_car_site","underground_car_site":"underground_car_site","parking_fee":"car_fee",
|
||||
// "elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","clear_height":"clear_height",
|
||||
// "foreign":"foreign","standard_area":"standard_area","foreign":"foreign","parking_fee":"car_fee","estate_company":"property_company","developer":"developer","all_area":"area",
|
||||
|
||||
static main(args) {
|
||||
// runBatch(new OfficeFieldAddBatch(), args)
|
||||
|
||||
def task = new OfficeFieldAddBatch()
|
||||
// task.fieldTempProcess()
|
||||
task.deleteNotNormalOffice()
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = ["深圳"]//CommonDatautils.findAllCity()
|
||||
def fields = fieldsMap.keySet() as List
|
||||
def analysRets = []
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
|
||||
|
||||
sources.each {source->
|
||||
def esRets = findAllEsDataByCity(c, source)
|
||||
def updateRets = processFieldsFromMatch(c , esRets, source)
|
||||
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
|
||||
def analyRet = []
|
||||
analyRet << source
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analysRets << analyRet
|
||||
}
|
||||
}
|
||||
println analysRets.toString()
|
||||
}
|
||||
|
||||
def processFieldsFromMatch(c,matchRets, source){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
def fieldValueMap = [:]
|
||||
for(def esOffice : matchRets){
|
||||
// if(esOffice.name!="东方建富时尚广场"){
|
||||
// continue
|
||||
// }
|
||||
def office = CommonDatautils.findNormalDistrict(esOffice.city, esOffice.region, esOffice.name, "office", fields)
|
||||
if(!office){
|
||||
// println c+" "+esOffice.name+" 不存在"
|
||||
continue
|
||||
}
|
||||
def newValues = strandField(c, office, esOffice, source)
|
||||
if(newValues){
|
||||
println office._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(office._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
// ["loop_line_info":"ring_location","feature":"characteristic","description":"introduction","lng":"location","open_date":"open_date"]
|
||||
def strandField( c, office, esOffice, source){
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = office.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = esOffice.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
|
||||
if(seaweedfield == "estate_type2"){
|
||||
newVal = FiledStrand.processEstateType(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), office.estate_type2)
|
||||
if(newVal) {
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
if(seaweedfield =="virescence_percent"){
|
||||
newVal = FiledStrand.processVirescencePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "loop_line_info"){
|
||||
newVal = FiledStrand.processLoopLine(esVal)
|
||||
}
|
||||
if(seaweedfield =="cubage_percent"){
|
||||
newVal = FiledStrand.processCubagePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area" || seaweedfield=="sight"){
|
||||
newVal = FiledStrand.processArea(esVal.toString())
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
|
||||
if(seaweedfield =="car_site_count" || seaweedfield =="land_car_site" || seaweedfield =="underground_car_site"){
|
||||
newVal = FiledStrand.processCarSite(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="property_life"){
|
||||
newVal = FiledStrand.processPropertyLife(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="address"){
|
||||
newVal = FiledStrand.processAddress(esVal, office.city)
|
||||
}
|
||||
if(seaweedfield =="block"){
|
||||
newVal = FiledStrand.processBlock(office.city, office.region, esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "fitment"){
|
||||
newVal = FiledStrand.processFitment(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_type"){
|
||||
newVal = FiledStrand.processBuildType(esVal, "office")
|
||||
}
|
||||
if(seaweedfield == "build_structure"){
|
||||
newVal = FiledStrand.processBuildStructure(esVal)
|
||||
}
|
||||
if(seaweedfield == "bed_kind"){
|
||||
newVal = FiledStrand.processLevel(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "total_floor" || seaweedfield == "land_floor"){
|
||||
newVal = FiledStrand.processFloor(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "underground_floor"){
|
||||
newVal = FiledStrand.processUndergroundFloor(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "real_use_percent"){
|
||||
newVal = FiledStrand.processRealUsePercent(esVal)
|
||||
}
|
||||
if(seaweedfield == "parking_fee"){
|
||||
newVal = FiledStrand.processCarSiteRent(esVal)
|
||||
println "[ ${source} ] old: "+esVal +" new:"+newVal
|
||||
}
|
||||
|
||||
if(seaweedfield == "elevators_count"){
|
||||
newVal = FiledStrand.processElevators(esVal)
|
||||
}
|
||||
if(seaweedfield == "person_elevators_count" && esVal.contains("客梯")){
|
||||
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("客梯")))
|
||||
}
|
||||
|
||||
if(seaweedfield == "goods_elevator_count" && esVal.contains("货梯")){
|
||||
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("货梯")))
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_height" || seaweedfield=="lobby_height" || seaweedfield=="clear_height"){
|
||||
newVal = FiledStrand.processStandardHeight(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_area"){
|
||||
newVal = FiledStrand.processStandardArea(esVal.toString())
|
||||
}
|
||||
|
||||
if(seaweedfield == "finish_building_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "foreign"){
|
||||
newVal = FiledStrand.processForeign(esVal)
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
if(seaweedfield == "lng"){
|
||||
try{
|
||||
def locationMap = FiledStrand.processLngAndLat(esVal)
|
||||
if(locationMap){
|
||||
newValues.put("lat", locationMap.lat)
|
||||
newValues.put("lng", locationMap.lng)
|
||||
newValues.put("lat2", locationMap.lat2)
|
||||
newValues.put("lng2", locationMap.lng2)
|
||||
}
|
||||
}catch(Exception e){
|
||||
println esVal
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return newValues;
|
||||
}
|
||||
|
||||
|
||||
def findAllEsDataByCity(city, source){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", source))
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "office2", queryBuilder);
|
||||
}catch(Exception e){
|
||||
datas = findAllEsDataByCity(city, source)
|
||||
}
|
||||
|
||||
return datas;
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
|
||||
for(def rm : updateRets){
|
||||
def valMap = rm.value
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
if(valMap.containsKey(seaweedfield)) {
|
||||
if(countMap.containsKey(seaweedfield)) {
|
||||
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
|
||||
}else{
|
||||
countMap.put(seaweedfield, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println countMap.toString()
|
||||
def precentMap = [:]
|
||||
fieldsMap.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
|
||||
}else {
|
||||
precentMap.put(key, 0)
|
||||
}
|
||||
}
|
||||
return precentMap;
|
||||
}
|
||||
|
||||
def updateDB(c, updateRets){
|
||||
|
||||
for(def rm : updateRets) {
|
||||
def id = rm.key
|
||||
def valMap = rm.value
|
||||
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
valMap.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def fieldTempProcess(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx",{d->
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("address", d['F'])
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
|
||||
})
|
||||
}
|
||||
|
||||
def deleteNotNormalOffice(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\需删除写字楼.xlsx",{d->
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("status", 1)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class OfficeStat {
|
||||
static main(args) {
|
||||
def task = new OfficeStat()
|
||||
// task.run()
|
||||
// task.sampleFieldPrecent()
|
||||
// task.cityRun()
|
||||
task.tempStat()
|
||||
}
|
||||
|
||||
def tempStat(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\写字楼租金统计.xlsx",{d->
|
||||
def cityObj = CommonDatautils.findAlCityDetail(d['A'])
|
||||
if(cityObj){
|
||||
if(cityObj.new_city_grade==1000){
|
||||
d['P'] = "一线"
|
||||
}else if(cityObj.new_city_grade==1500){
|
||||
d['P'] = "新一线"
|
||||
}else if(cityObj.new_city_grade==2000){
|
||||
d['P'] = "二线"
|
||||
}else if(cityObj.new_city_grade==3000){
|
||||
d['P'] = "三线"
|
||||
}else if(cityObj.new_city_grade==4000){
|
||||
d['P'] = "四线"
|
||||
}else if(cityObj.new_city_grade==5000){
|
||||
d['P'] = "五线"
|
||||
}
|
||||
}
|
||||
// def count3s = []
|
||||
// if(d['C']>=3){
|
||||
// count3s << d['C']
|
||||
// }
|
||||
// if(d['D']>=3 ){
|
||||
// count3s << d['D']
|
||||
// }
|
||||
// if(d['E']>=3 ){
|
||||
// count3s << d['E']
|
||||
// }
|
||||
// if(d['F']>=3 ){
|
||||
// count3s << d['F']
|
||||
// }
|
||||
// if(d['G']>=3 ){
|
||||
// count3s << d['G']
|
||||
// }
|
||||
// if(d['H']>=3 ){
|
||||
// count3s << d['H']
|
||||
// }
|
||||
// if(d['I']>=3 ){
|
||||
// count3s << d['I']
|
||||
// }
|
||||
// if(d['J']>=3 ){
|
||||
// count3s << d['J']
|
||||
// }
|
||||
// if(d['K']>=3 ){
|
||||
// count3s << d['K']
|
||||
// }
|
||||
// if(d['L']>=3 ){
|
||||
// count3s << d['L']
|
||||
// }
|
||||
// if(d['M']>=3 ){
|
||||
// count3s << d['M']
|
||||
// }
|
||||
// if(d['N']>=3 ){
|
||||
// count3s << d['N']
|
||||
// }
|
||||
// d['O'] = DataHandleUtils.getDoubleRound(4, count3s.size()/12)
|
||||
},2)
|
||||
}
|
||||
|
||||
def sampleFieldPrecent(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","级别","面积","最高楼","装修","单价(元/平米/天)","发布时间","总量"]
|
||||
citys.each { c ->
|
||||
def regionMap = CommonDatautils.getRegionsByCity(c)
|
||||
def ret = sampleFieldStat(c, regionMap, "office_rent")
|
||||
rets.addAll(ret)
|
||||
}
|
||||
///home/goojia/gj
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼租金样本字段覆盖率")
|
||||
}
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
rets << ["城市","区域","近1个月样本量","近2个月样本量","近3个月样本量","近4个月样本量","近5个月样本量","近6个月样本量","近7个月样本量","近8个月样本量","近9个月样本量","近10个月样本量","近11个月样本量","近12个月样本量","样本覆盖率(区域走势)" ]
|
||||
citys.each{c->
|
||||
def regionMap = CommonDatautils.getRegionsByCity(c)
|
||||
def ret = statHouse(c, regionMap, "office_rent")
|
||||
rets.addAll(ret)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼区域租金样本覆盖率")
|
||||
}
|
||||
|
||||
def cityRun(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
rets << ["城市","区域","近1个月样本量","近2个月样本量","近3个月样本量","近4个月样本量","近5个月样本量","近6个月样本量","近7个月样本量","近8个月样本量","近9个月样本量","近10个月样本量","近11个月样本量","近12个月样本量","样本覆盖率(区域走势)" ]
|
||||
citys.each{c->
|
||||
def ret = statCityHouse(c, "office_rent")
|
||||
rets.addAll(ret)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\写字楼租金统计.xlsx", rets, "写字楼城市租金样本覆盖率")
|
||||
}
|
||||
|
||||
def statHouse(city, regionMap, cat){
|
||||
def regions = regionMap.keySet()
|
||||
def rows = []
|
||||
def months = [20250201, 20250101,20241201,20241101,20241001,20240901,20240801,20240701,20240601,20240501,20240401,20240301,20240201]
|
||||
regions.each{region->
|
||||
def row = []
|
||||
row << city
|
||||
row << region
|
||||
|
||||
def count3s = []
|
||||
for(def i=1;i<months.size();i++){
|
||||
def count = findHouseCountByCat(city, region, cat, months[i], months[0])
|
||||
row << count
|
||||
if(count>=3){
|
||||
count3s << count
|
||||
}
|
||||
}
|
||||
def precent = DataHandleUtils.getDoubleRound(4, count3s.size()/12)
|
||||
row << precent
|
||||
println row
|
||||
rows << row
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
def statCityHouse(city, cat){
|
||||
def months = [20250201, 20250101,20241201,20241101,20241001,20240901,20240801,20240701,20240601,20240501,20240401,20240301,20240201]
|
||||
def row = []
|
||||
row << city
|
||||
|
||||
def count3s = []
|
||||
for(def i=1;i<months.size();i++){
|
||||
def count = findHouseCountByCat(city, null, cat, months[i], months[i-1])
|
||||
row << count
|
||||
if(count>=3){
|
||||
count3s << count
|
||||
}
|
||||
}
|
||||
def precent = DataHandleUtils.getDoubleRound(4, count3s.size()/12)
|
||||
row << precent
|
||||
println row
|
||||
return row
|
||||
}
|
||||
|
||||
def sampleFieldStat(city, regionMap, cat){
|
||||
def regions = regionMap.keySet()
|
||||
def rows = []
|
||||
regions.each{region->
|
||||
def results = findHouseByCat(city, region, cat, 20240201,20250201)
|
||||
def row = []
|
||||
def total = results.size()
|
||||
def nameCount = 0
|
||||
def levelCount = 0
|
||||
def heightCount = 0
|
||||
def areaCount = 0
|
||||
def priceCount = 0
|
||||
def dateCount = 0
|
||||
def fitmentCount = 0
|
||||
row << city
|
||||
row << region
|
||||
results.each{m->
|
||||
def name = m.get("name")
|
||||
if(name){
|
||||
nameCount++
|
||||
}
|
||||
|
||||
def level = m.get("level")
|
||||
if(level){
|
||||
levelCount++
|
||||
}
|
||||
|
||||
def height = m.get("height")
|
||||
if(height){
|
||||
heightCount++
|
||||
}
|
||||
|
||||
def fitment = m.get("fitment")
|
||||
if(fitment){
|
||||
fitmentCount++
|
||||
}
|
||||
|
||||
def area = m.get("area")
|
||||
if(area){
|
||||
areaCount++
|
||||
}
|
||||
def day_rent_price = m.get("day_rent_price")
|
||||
if(day_rent_price){
|
||||
priceCount++
|
||||
}
|
||||
|
||||
def p_date = m.get("p_date")
|
||||
if(p_date){
|
||||
dateCount++
|
||||
}
|
||||
}
|
||||
if(total){
|
||||
row << DataHandleUtils.getDoubleRound(4, nameCount/total)
|
||||
row << DataHandleUtils.getDoubleRound(4, levelCount/total)
|
||||
row << DataHandleUtils.getDoubleRound(4, areaCount/total)
|
||||
row << DataHandleUtils.getDoubleRound(4, heightCount/total)
|
||||
row << DataHandleUtils.getDoubleRound(4, fitmentCount/total)
|
||||
row << DataHandleUtils.getDoubleRound(4, priceCount/total)
|
||||
row << DataHandleUtils.getDoubleRound(4, dateCount/total)
|
||||
}else{
|
||||
row << 0
|
||||
row << 0
|
||||
row << 0
|
||||
row << 0
|
||||
row << 0
|
||||
row << 0
|
||||
row << 0
|
||||
}
|
||||
|
||||
row << total
|
||||
rows << row
|
||||
println row
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
def findHouseCountByCat(def city, def region, def cat, def startDate, def endDate){
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
if(region){
|
||||
sql = sql + " and region='${region}'"
|
||||
}
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
def findHouseByCat(def city, def region, def cat, def startDate, def endDate){
|
||||
|
||||
String sql = "select * from latest_seawater where cat='${cat}' and city='${city}' and region='${region}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
def results =[]
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
results << ["level":data.level,"name":data.district_name,"day_rent_price":data.total_price,"height":data.height,"area":data.area,"p_date":data.p_date,"fitment":data.fitment]
|
||||
})
|
||||
return results
|
||||
}
|
||||
}
|
||||
+8
-7
@@ -5,6 +5,7 @@ import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -25,7 +26,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -12, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
@@ -52,11 +53,11 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(["all"])
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
@@ -519,7 +520,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20250131", "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user