From eb80d9401f9994f6ae19203f791003ba3e81f845 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 10:23:50 +0800 Subject: [PATCH 01/50] update --- .../ronghe/ExportDistrictRongHeTest.groovy | 50 +-- .../com/fangjia/script/gj/EsHouseStat.groovy | 20 +- .../com/fangjia/script/gj/FiledStrand.groovy | 78 ++++- .../gj/district/DistrictFieldAddBatch.groovy | 320 ++++++++++++------ .../gj/district/DistrictFiledStat.groovy | 69 ++++ .../script/gj/es/EsMonthHouseStat.groovy | 39 ++- .../gj/paimai/BeiKeCrawlByChrome.groovy | 12 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 6 +- .../script/trade/OpenUriUserIdCountDay.groovy | 10 +- 9 files changed, 454 insertions(+), 150 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 0f2519b79..66f2cc491 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -36,35 +36,35 @@ class ExportDistrictRongHeTest extends BaseBatch{ // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) -// if(curDate.toString().endsWith("10")){ + if(curDate.toString().endsWith("10")){ -// exportEvaluateRate() -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// -// -// exportBigdataCommunityGJRH()/ -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// -// -// -// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()/ + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) exportInOk() -// } + } SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy index b78e0ec4b..da0c07889 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy @@ -29,16 +29,16 @@ class EsHouseStat { DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) // DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020) -// public static JdbcDataSource dataSource = new JdbcDataSource() -// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" -// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) -// static { -// dataSource.setUrl(address) -// Properties connectionProperties = new Properties() -// connectionProperties.put("user", "developer") -// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") -// dataSource.setProperties(connectionProperties) -// } + public static JdbcDataSource dataSource = new JdbcDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 805c41944..e6cdef564 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -285,7 +285,7 @@ class FiledStrand { } static def processCarSiteDesc(car_site){ - if(car_site && car_site.size()<=20){ + if(car_site && car_site.size()<=20 && car_site.size()>=2 && !car_site.contains("待定") && !car_site.contains("暂无")){ return car_site.toString() } return null @@ -337,6 +337,82 @@ class FiledStrand { return null } + static def processBlock(def block){ + if(block){ + if(block.contains("无") || block.toString().length()>10){ + return null + } + return block + } + return null + } + + static def processBuildType(def buildType, def estateType){ + if(buildType){ + def types = [] + if(estateType == "别墅"){ + def arrs = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"] + for(def s : arrs){ + if(buildType.contains(s)){ + types << s + } + } + }else if(estateType == "商住"){ + def arrs = ["独栋","双拼","联排","叠加","老式别墅洋房"] + for(def s : arrs){ + if(buildType.contains(s)){ + types << s + } + } + }else{ + def arrs = ["多层","小高层","高层","超高层","SOHO","soho","LOFT","loft","酒店式公寓"] + for(def s : arrs){ + if(buildType.contains(s)){ + types << s + } + } + } + if(types){ + return types.join(",").toString() + } + } + return null + } + + static def processlandProperty(val){ + if(val){ + def arrs = ["经适房","限价房","廉租房","宅基地","保障房","动迁房","配套房","商品房"] + def property = null + for(def s : arrs){ + if(val.contains(s)){ + property = s + break; + } + } + return property + } + return null + } + + static def processBuildStructure(val) { + if (val) { + def buildStructure= null + if(val.indexOf("砖木")>-1){ + buildStructure = "砖木结构" + }else if(val.indexOf("钢结构")>-1){ + buildStructure = "钢结构"; + }else if(val.indexOf("钢、钢筋混凝土")>-1){ + buildStructure = "钢、钢筋混凝土结构"; + }else if(val.indexOf("钢筋混凝土")>-1){ + buildStructure = "钢筋混凝土结构"; + }else if(val.indexOf("混合")>-1){ + buildStructure = "混合结构"; + } + return buildStructure + } + return null + } + static def isRoad(def s){ def pattern = Pattern.compile("[路|街|大道|巷]"); def matcher = pattern.matcher(s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index d32f3e459..5495b6bfd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -7,9 +7,11 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.FiledStrand import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.Jms import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -17,6 +19,7 @@ import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import org.bson.types.ObjectId +import java.text.SimpleDateFormat import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -31,15 +34,32 @@ import java.util.regex.Pattern class DistrictFieldAddBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) - + static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018) //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" - //,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家" - def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源 + // + def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", + "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源 //"alias":"alias", - def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段 + //ownership_type:权属类型 为商品房、经适房、动迁房等 + //house_property 产权性质:国有产权 "house_property":"网站没有", + def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", + "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", + "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", + "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", + "loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段 + + def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商", + "virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型", + "car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构", + "build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"] + + + final static def EMAILS = [ + "gaojuan@fangjia.com" + ] static main(args) { runBatch(new DistrictFieldAddBatch(), args) @@ -47,21 +67,37 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京" - + def citys = CommonDatautils.findAllCity() def analysRets = [] + + def rh_analysRets = [] + def fields = fieldsMap.keySet() as List + def headers = ["来源", "城市"] + fields.each{it-> + headers << fieldsNameMap.get(it) + } + + headers << "抓取量" + headers << "匹配量" + headers << "0.9分以上量" + headers << "匹配率" + headers << "0.9以上匹配率" +// analysRets << headers + for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) -// addFieldFromRonghe(c, totalCount) + rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) analysRets = addFieldFromMatch(c, totalCount, analysRets) } println analysRets.toString() - - ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据") + sendStatEmail(headers, analysRets, rh_analysRets) +// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据") } def addFieldFromMatch(c, totalCount, analysRets){ + def fields = fieldsMap.keySet() as List + sourceMap.each { seaweedmapSource, esWebSource -> def esRets = findEsDatas(c, esWebSource) def matchRets1 = matchApi(esRets) @@ -80,7 +116,18 @@ class DistrictFieldAddBatch extends BaseBatch{ pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size()) pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size()) } - analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2] + def analyRet = [] + analyRet << seaweedmapSource + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analyRet << matchRets1.size() + analyRet << matchRets2.size() + analyRet << pre1 + analyRet << pre2 + analysRets << analyRet } return analysRets; } @@ -88,7 +135,8 @@ class DistrictFieldAddBatch extends BaseBatch{ - def addFieldFromRonghe(c, totalCount){ + def addFieldFromRonghe(c, totalCount, rh_analysRets){ + def fields = fieldsMap.keySet() as List sourceMap.each{seaweedmapSource,esWebSource-> def rhRets = findRongheDists(c, seaweedmapSource) def esRets = findEsDistrict(c, esWebSource) @@ -99,7 +147,26 @@ class DistrictFieldAddBatch extends BaseBatch{ println c +" "+precentMap.toString() updateDB(c, updateRets) println "----------------------------------------------------------------------------------" + def pre1 = 0 + def pre2 = 0 + if(esRets){ + pre1 = DataHandleUtils.getDoubleRound(2, rhRets.size()*100/esRets.size()) + pre2 = DataHandleUtils.getDoubleRound(2, esRets.size()*100/esRets.size()) + } + def analyRet = [] + analyRet << seaweedmapSource + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analyRet << rhRets.size() + analyRet << esRets.size() + analyRet << pre1 + analyRet << pre2 + rh_analysRets << analyRet } + return rh_analysRets; } def processFieldsFromRh(c, rhRets, esRets){ @@ -116,26 +183,7 @@ class DistrictFieldAddBatch extends BaseBatch{ println c+" "+rhDist.id+" 不存在[ES]" continue } - def newValues = [:] - fieldsMap.each{seaweedfield,esField-> - def val = dist.get(seaweedfield) - if(!val){ - def esVal = esDbo.get(esField) - if(esVal){ - def newVal = null - if(seaweedfield == "estate_charge"){ - newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) - newVal = newVal.toString() - } - if(seaweedfield == "build_date"){ - newVal = FiledStrand.processFinishBuildingDate(esVal) - } - if(newVal){ - newValues.put(seaweedfield, newVal) - } - } - } - } + def newValues = strandField(c, dist, esDbo) if(newValues){ fieldValueMap.put(dist._id.toString(), newValues) } @@ -149,77 +197,15 @@ class DistrictFieldAddBatch extends BaseBatch{ def processFieldsFromMatch(c,matchRets){ def fields = fieldsMap.keySet() as List fields << "city" + fields << "estate_type2" def fieldValueMap = [:] for(def matchDist : matchRets){ - def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) if(!dist){ println c+" "+matchDist.match_id+" 不存在" continue } - - def newValues = [:] - fieldsMap.each{seaweedfield,esField-> - def val = dist.get(seaweedfield) - if(!val){ - def esVal = matchDist.get(esField) - if(esVal){ - def newVal = null - if(seaweedfield == "estate_charge"){ - newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) - newVal = newVal.toString() - } - if(seaweedfield == "build_date"){ - newVal = FiledStrand.processFinishBuildingDate(esVal) - } - if(seaweedfield == "estate_company" || seaweedfield == "developer"){ - newVal = FiledStrand.processCompany(esVal) - } - if(seaweedfield =="virescence_percent"){ - newVal = FiledStrand.processVirescencePercent(esVal) - } - - if(seaweedfield =="cubage_percent"){ - newVal = FiledStrand.processCubagePercent(esVal) - } - - if(seaweedfield =="all_area" || seaweedfield =="build_area"){ - newVal = FiledStrand.processArea(esVal) - newVal = newVal.toString() - } - if(seaweedfield =="household_count"){ - newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2) - } - - if(seaweedfield =="loudong_count"){ - newVal = FiledStrand.processLoudongCount(esVal) - } - - if(seaweedfield =="car_site_count"){ - newVal = FiledStrand.processCarSite(esVal) - } - if(seaweedfield =="car_site"){ - newVal = FiledStrand.processCarSiteDesc(esVal) - } - - if(seaweedfield =="property_life"){ - newVal = FiledStrand.processPropertyLife(esVal) - } - - if(seaweedfield =="alias"){ - newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias) - } - - if(seaweedfield =="address"){ - newVal = FiledStrand.processAddress(esVal, dist.city) - } - - if(newVal){ - newValues.put(seaweedfield, newVal) - } - } - } - } + def newValues = strandField(c, dist, matchDist) if(newValues){ println dist._id.toString()+" "+newValues.toString() fieldValueMap.put(dist._id.toString(), newValues) @@ -229,6 +215,89 @@ class DistrictFieldAddBatch extends BaseBatch{ return fieldValueMap; } + def strandField(c,dist, matchDist){ + def newValues = [:] + fieldsMap.each{seaweedfield,esField-> + def val = dist.get(seaweedfield) + if(!val){ + def esVal = matchDist.get(esField) + if(esVal){ + def newVal = null + if(seaweedfield == "estate_charge"){ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) + newVal = newVal.toString() + } + if(seaweedfield == "build_date" || seaweedfield == "live_date"){ + newVal = FiledStrand.processFinishBuildingDate(esVal) + } + if(seaweedfield == "estate_company" || seaweedfield == "developer"){ + newVal = FiledStrand.processCompany(esVal) + } + if(seaweedfield =="virescence_percent"){ + newVal = FiledStrand.processVirescencePercent(esVal) + } + + if(seaweedfield =="cubage_percent"){ + newVal = FiledStrand.processCubagePercent(esVal) + } + + if(seaweedfield =="all_area" || seaweedfield =="build_area"){ + newVal = FiledStrand.processArea(esVal) + if(newVal){ + newVal = newVal.toString() + } + } + if(seaweedfield =="household_count"){ + newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2) + } + + if(seaweedfield =="loudong_count"){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(seaweedfield =="car_site_count"){ + newVal = FiledStrand.processCarSite(esVal) + } + if(seaweedfield =="car_site"){ + newVal = FiledStrand.processCarSiteDesc(esVal) + } + + if(seaweedfield =="property_life"){ + newVal = FiledStrand.processPropertyLife(esVal) + } + + if(seaweedfield =="alias"){ + newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias) + } + + if(seaweedfield =="address"){ + newVal = FiledStrand.processAddress(esVal, dist.city) + } + if(seaweedfield =="block"){ + newVal = FiledStrand.processBlock(esVal) + } + + if(seaweedfield == "build_type"){ + newVal = FiledStrand.processBuildType(esVal, dist.estate_type2) + } + + if(seaweedfield == "land_property"){ + newVal = FiledStrand.processlandProperty(esVal) + } + + if(seaweedfield == "build_structure"){ + newVal = FiledStrand.processBuildStructure(esVal) + } + + if(newVal){ + newValues.put(seaweedfield, newVal) + } + } + } + } + return newValues; + } + def findRongheDists(def city, def source){ @@ -285,9 +354,9 @@ class DistrictFieldAddBatch extends BaseBatch{ def precentMap = [:] fieldsMap.each() {key,skey-> if(countMap.containsKey(key)) { - precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) + precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) }else { - precentMap.put(skey, 0) + precentMap.put(key, 0) } } return precentMap; @@ -414,4 +483,61 @@ class DistrictFieldAddBatch extends BaseBatch{ } return false } + + def sendStatEmail(headers, analysRets, rh_nalysRets){ + Date dateTime = DateUtils.mins(1); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd"); + StringBuffer sb = new StringBuffer(); + if(analysRets){ + sb.append("") + sb.append("字段补充情况分析") + sb.append("") + headers.each{it-> + if(it=="来源" || it=="城市"){ + sb.append("") + }else{ + sb.append("") + } + } + sb.append("") + + for(def ret : analysRets){ + sb.append(""); + ret.each{it-> + sb.append("") + } + sb.append(""); + } + sb.append("
"+ it+""+ it+"
" + it + "

"); + } + + + StringBuffer rh_sb = new StringBuffer(); + if(rh_nalysRets){ + rh_sb.append("") + rh_sb.append("字段补充情况分析") + rh_sb.append("") + headers.each{it-> + if(it=="来源" || it=="城市"){ + rh_sb.append("") + }else{ + rh_sb.append("") + } + } + rh_sb.append("") + + for(def ret : rh_nalysRets){ + rh_sb.append(""); + ret.each{it-> + rh_sb.append("") + } + rh_sb.append(""); + } + rh_sb.append("
"+ it+""+ it+"
" + it + "

"); + } + + for (def email : EMAILS) { + jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString()); + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy new file mode 100644 index 000000000..47f1563d0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy @@ -0,0 +1,69 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 字段提升率统计 + */ +DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + +def fieldsMap = ["district_count":"小区数量","name":"名称" ,"region":"区域" ,"block":"板块" ,"address":"地址" ,"estateType2":"物业类型" ,"householdCount":"户数" ,"openDate":"开盘时间" ,"liveDate":"入住时间" ,"buildDate":"建筑年代" ,"allArea":"占地面积" ,"buildArea":"建筑面积" ,"buildType":"建筑类型" ,"developer":"开发商" ,"estateCompany":"物业公司" ,"cubagePercent":"容积率" ,"virescencePercent":"绿化率" ,"carSite":"停车位" ,"estateCharge":"物业费" ,"floorplanPics":"户型图" ,"outPics":"外景图" ,"finishBuildingDate":"竣工时间" ,"businessQuarter":"商圈" ,"startBuildingDate":"开建时间" ,"loopLineInfo":"环线" ,"propertyLife":"产权年限" ,"houseProperty":"产权性质" ,"licence":"预售许可证" ,"buildStructure":"建筑结构" ,"lat":"百度纬度" ,"pics":"图片" ,"streetCommunity":"街道" ,"schools":"学校" ,"houseStructure":"房屋结构" ,"alias":"别名" ,"addrAlias":"地址别名" ,"mainTypeArea":"主力面积" ,"loudongCount":"楼栋数" ,"latLng" :"高德经度" ,"rentPrice":"租金" ] +def weeknd = DateUtils.getFirstDayOfWeek(new Date()) +def lastWeeknd = DateUtils.getFirstDayOfWeek(new Date()-7) + + +def weekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", weeknd)) +def lastWeekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", lastWeeknd)) + +def weekendMapDatas = findDataByCity(weekendDatas) +def lastWeekendMapDatas = findDataByCity(lastWeekendDatas) + +def citys = CommonDatautils.findAllCity() + +def fields = fieldsMap.keySet() as List +def rets = [] +def headers = [] +headers << "城市" +fields.each{k-> + headers << fieldsMap.get(k) +} +rets << headers +for(def c : citys){ + def weekendDetailMap = weekendMapDatas.get(c) + def lastWeekendDetailMap = lastWeekendMapDatas.get(c) + def ret = [] + ret << c + fields.each{key-> + if(weekendDetailMap){ + def v1 = weekendDetailMap.get(key) + def v2 = lastWeekendDetailMap.get(key) + if(v1 && v2){ + ret << DataHandleUtils.getDoubleRound(2, v1-v2) + }else{ + ret << "-" + } + }else{ + ret << "-" + } + } + rets << ret +} + +ExcelUtils.write("D:\\数据\\小区\\字段提升率"+DateUtils.formatDateByPattern(weeknd,"yyyy-MM-dd")+".xlsx", rets, "字段提升率统计") + + + +def findDataByCity(weekendDatas){ + def map = [:] + for(def d : weekendDatas){ + d.district_detail.district_count = d.existing_district_count + map.put(d.city, d.district_detail) + } + return map +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index a3e22d7a6..9be75aeb0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.gj.es +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils @@ -19,11 +21,44 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() - task.houseRun() +// task.houseRun() // task.statHouseUpdateCount() // task.statDistHouse() + task.statRent() + } + + def statRent(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\其它\\租金.xlsx", { d -> + index++ + def region = RegionData.formatRegion(d['A'], d['B']) + if(region){ + def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(!dist){ + dist = OpenApiUtils.matchDistrictByAddr(d['A'], d['C'], region) + if(dist){ + dist = CommonDatautils.findDistrictById(dist.id) + } + } + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + def distCount = findHouseCount(dist.city, dist.region, alias, 20220501, 20230501) + d['G'] = distCount + d['E'] = dist.region + d['F'] = dist.name + println index+" "+ d['C'] +" "+distCount + } + }); } def statDistHouse(){ @@ -172,7 +207,7 @@ class EsMonthHouseStat { def total = 0 try{ aliasStr = escapeExprSpecialWord(alias.join("','")) - String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + String sql = "select count(*) as c from latest_seawater where cat='rent' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and root='lianjia.com'" ElasticUtils.executeSqlQuery(sql,{data-> total = data.c diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 2aae0edb5..916db9961 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -239,6 +239,12 @@ public class BeiKeCrawlByChrome { if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } + if(!detail.area){ + areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "") + if (areaStr && areaStr.isNumber()) { + detail.area = areaStr.toDouble() + } + } def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { @@ -254,7 +260,7 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" - if(detail.total_price){ + if(detail.total_price && detail.avg_price){ def detailUrl = e.select(".title a").attr("href") detail.url = detailUrl def dbo = findTrade(detail) @@ -404,9 +410,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") - if (citys.contains(e.text())){ +// if (citys.contains(e.text())){ list << [e.text(), href] - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 343a4a3a4..6c01e6c11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -333,7 +333,7 @@ public class LianjiaCrawlByChrome { detail.source = "链家在线" detail.url = e.select(".title a").attr("href") - if(detail.total_price){ + if(detail.total_price && detail.avg_price){ def dbo = findTrade(detail) if (!dbo) { def houseInfo = e.select(".houseInfo").text() @@ -482,9 +482,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ +// if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] - } +// } } } return cityList diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy index 4e27d62a5..954fface3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy @@ -27,19 +27,11 @@ class OpenUriUserIdCountDay extends BaseBatch { runBatch(new OpenUriUserIdCountDay(), args) } final static def EMAILS = [ - "joe@fangjia.com", "wangwenyan@fangjia.com", "xiaohai@fangjia.com", "caoyuyan@fangjia.com", "jiangxiaochun@fangjia.com", - "gaojuan@fangjia.com", - 'shijing@fangjia.com', - 'zhongjingyi@fangjia.com', - "duweiliang@fangjia.com", - "sunjian@fangjia.com", - "yaojingmeng@fangjia.com", - "xuqi@fangjia.com", - "wangpeili@fangjia.com" + "gaojuan@fangjia.com" ] @Override From 0e26de8cad44e2273c099232f52019fb41e30c4e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 10:25:43 +0800 Subject: [PATCH 02/50] update --- .../com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 5495b6bfd..7c89352be 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity().reverse() def analysRets = [] From b9780aae68ee939af5cd3179878b8d8b7c12d471 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 11:10:01 +0800 Subject: [PATCH 03/50] update --- .../fangjia/script/gj/district/DistrictFieldAddBatch.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 7c89352be..f3a47806d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -87,8 +87,10 @@ class DistrictFieldAddBatch extends BaseBatch{ for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) - rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) - analysRets = addFieldFromMatch(c, totalCount, analysRets) + if(totalCount){ + rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) + analysRets = addFieldFromMatch(c, totalCount, analysRets) + } } println analysRets.toString() sendStatEmail(headers, analysRets, rh_analysRets) From e441a10add801c62b94ab3255ab9622ccb82dd6c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 18:36:11 +0800 Subject: [PATCH 04/50] update --- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../gj/district/DistrictFieldAddBatch.groovy | 5 ++- .../gj/paimai/BeiKeCrawlByChrome.groovy | 44 ++++++++++++++----- .../gj/paimai/LianjiaCrawlByChrome.groovy | 31 ++++++++++--- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index e26dc05dd..a3be166aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index f3a47806d..2806a48f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -88,7 +88,7 @@ class DistrictFieldAddBatch extends BaseBatch{ for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) if(totalCount){ - rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) +// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) analysRets = addFieldFromMatch(c, totalCount, analysRets) } } @@ -175,6 +175,9 @@ class DistrictFieldAddBatch extends BaseBatch{ def fields = fieldsMap.keySet() as List def fieldValueMap = [:] for(def rhDist : rhRets){ + if(rhDist.commid && rhDist.commid!="null"){ + continue + } def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields) if(!dist){ println c+" "+rhDist.commid+" 不存在" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 916db9961..ca92bb2c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome { // // } // citys = citys[(start+1)..(citys.size()-1)] - getLogin(driver) + driver = getLogin() crawlData(driver, citys) @@ -108,15 +108,32 @@ public class BeiKeCrawlByChrome { } } + def checkDocument(driver, url){ + def s = driver.getPageSource() + def flag = true + while(flag){ + if(s && s.contains("无法访问此网站")){ + driver = getLogin() + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + } + return Jsoup.parse(s); + } + def crawlData(driver, citys){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) for(def m : citys){ def city = m[0] def url = m[1] - driver.get(url+"/chengjiao/") + def newUrl = url+"/chengjiao/" + driver.get(newUrl) Thread.sleep(10000) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, newUrl); def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -220,6 +237,9 @@ public class BeiKeCrawlByChrome { } def roomStr = arrs[1] + if(roomStr && !roomStr.contains("室")){ + roomStr = arrs[2] + } if (roomStr) { if (roomStr.indexOf("室") > 0) { roomStr = roomStr.substring(0, roomStr.indexOf("室")) @@ -235,16 +255,16 @@ public class BeiKeCrawlByChrome { } } - def areaStr = arrs[2].replace("平米", "") + def areaStr = arrs[2] + if(areaStr && !areaStr.contains("平米")){ + if(arrs.size()>=4){ + areaStr = arrs[3] + } + } + areaStr = areaStr.replace("平米","") if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } - if(!detail.area){ - areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "") - if (areaStr && areaStr.isNumber()) { - detail.area = areaStr.toDouble() - } - } def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { @@ -418,7 +438,8 @@ public class BeiKeCrawlByChrome { return list } - def getLogin(driver){ + def getLogin(){ + WebDriver driver = new ChromeDriver() driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) @@ -438,6 +459,7 @@ public class BeiKeCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(10000) + return driver } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 6c01e6c11..bc5448c6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -122,7 +122,7 @@ public class LianjiaCrawlByChrome { WebDriver driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) // cityMap = cityMap.reverse() - getLogin(driver) + driver = getLogin() crawlData(driver, cityMap) @@ -191,10 +191,12 @@ public class LianjiaCrawlByChrome { for(def ret : cityList){ def city = ret[0] def cityUrl = ret[1] + def newUrl = cityUrl+"chengjiao/" + driver.get(newUrl) //def detailUrls = getPageUrls(driver, cityUrl) - driver.get(cityUrl+"chengjiao/") Thread.sleep(10000) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, newUrl); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -240,6 +242,21 @@ public class LianjiaCrawlByChrome { return detailUrls } + def checkDocument(driver, url){ + def s = driver.getPageSource() + def flag = true + while(flag){ + if(s && s.contains("无法访问此网站")){ + driver = getLogin() + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + } + return Jsoup.parse(s); + } def getPageDeail(driver, city, cityUrl, regionText){ def stop = false @@ -278,7 +295,7 @@ public class LianjiaCrawlByChrome { def parse(city,e, regionText){ try { - def title = e.select(".title a").text() + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() def arrs = title.split(" ") @@ -318,7 +335,7 @@ public class LianjiaCrawlByChrome { } } - def price = e.select(".totalPrice .number").text() + def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { detail.total_price = (int)(price.toDouble() * 10000) } @@ -490,7 +507,8 @@ public class LianjiaCrawlByChrome { return cityList } - def getLogin(driver){ + def getLogin(){ + WebDriver driver = new ChromeDriver() driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) @@ -510,6 +528,7 @@ public class LianjiaCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(20000) + return driver } } From 25c5837a26faaa253714d70327541d2218d594db Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 15 May 2023 11:36:31 +0800 Subject: [PATCH 05/50] =?UTF-8?q?=E6=9D=BF=E5=9D=97=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-bkoff/src/main/webapp/district/add.ftl | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fangjia-bkoff/src/main/webapp/district/add.ftl b/projects/fangjia-bkoff/src/main/webapp/district/add.ftl index 98b9daf40..e7c1e520b 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/add.ftl @@ -704,6 +704,7 @@ $(document).ready(function(){ success: function(model) { $("#block option").remove(); if (model.blockList.length > 0) { + $("").appendTo($("#block")); $.each(model.blockList, function(i, block){ $("").appendTo($("#block")); }); From fca5172971820ea67aec4ab3156a14f9642abac8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 15 May 2023 15:25:59 +0800 Subject: [PATCH 06/50] =?UTF-8?q?=E6=96=B0=E8=AF=A6=E6=83=85=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 105 ++++++++++++++++++ .../com/fangjia/open/dto/DistrictInfoDto.java | 9 ++ 2 files changed, 114 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index cdaa46a50..5e3c7c3d7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -822,6 +822,111 @@ public class PropertyController { return mav; } + + public Object detail2Action(PropertySearchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getId()) + && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getDistrict()) + || StringUtils.isBlank(model.getName())) + && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getDistrict()) + || StringUtils.isBlank(model.getAddress()))) { + mav.setModel(BasicModel.buildBadRequestModel("参数非法:请输入正确的参数组合")); + return mav; + } + + District district = null; + if (StringUtils.isNotBlank(model.getId())){ + if(ObjectId.isValid(model.getId())){ + district = districtService.findById(model.getId()); + }else{ + mav.setModel(BasicModel.buildBadRequestModel("非法的住宅ID")); + return mav; + } + } + + if (district == null && StringUtils.isNotBlank(model.getCity()) && StringUtils.isNotBlank(model.getDistrict()) + && StringUtils.isNotBlank(model.getName())) { + district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(), + "district"); + if (district != null) { + district = districtService.findById(district.getId()); + } + } + + if (district == null && StringUtils.isNotBlank(model.getCity()) && StringUtils.isNotBlank(model.getDistrict()) + && StringUtils.isNotBlank(model.getAddress())) { + Map result = addressParser.parse2(model.getCity(), model.getAddress(), "district", false); + saveAddressMatchLog(result, model.getAddress(), model.getCity(), model.getToken()); + Integer status = (Integer) result.get("status"); + Integer precise = (Integer) result.get("precise"); + if (status != null && status == 0 && precise == 0) { + String id = (String) result.get("id"); + district = districtService.findById(id); + } + } + + if (district != null) { + fieldFilterService.filterWithProcessNUll(district); + DistrictInfoDto districtInfoDto = new DistrictInfoDto(); + districtInfoDto.setId(district.getId()); + districtInfoDto.setCity(district.getCity()); + districtInfoDto.setDistrict(district.getRegion()); + districtInfoDto.setName(district.getName()); + districtInfoDto.setAlias(district.getMappingAlias()); + districtInfoDto.setAddress(district.getAddress()); + districtInfoDto.setAddressAlias(district.getAddrAlias()); + if (district.getLat2() != null && district.getLat2() != 0 && district.getLng2() != null + && district.getLng2() != 0) { + districtInfoDto.setLat(district.getLat2() / 10000000000D); + districtInfoDto.setLng(district.getLng2() / 10000000000D); + } + districtInfoDto.setPropertyType(district.getEstateType2()); + if (HouseValidator.validateBuildingType(district.getBuildType())) { + districtInfoDto.setBuildingType(district.getBuildType()); + } else { + districtInfoDto.setBuildingType(""); + } + districtInfoDto.setBlock(district.getBlock()); + districtInfoDto.setTownship(district.getTownship()); + if (district.getLoudongCount() != null && district.getLoudongCount() > 0) { + districtInfoDto.setBuildingCount(district.getLoudongCount()); + } + + if (district.getHouseholdCount() != null) { + districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString())); + } else { + districtInfoDto.setTotalUnitCount(""); + } + districtInfoDto.setManagementFee(district.getEstateCharge()); + if (district.getCarSiteCount() != null) { + districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString())); + } else { + districtInfoDto.setParking(""); + } + districtInfoDto.setOwnershipPeriod(district.getPropertyLife()); + districtInfoDto.setOwnershipType(district.getHouseProperty()); + districtInfoDto.setBuildingStructure(district.getBuildStructure()); + districtInfoDto.setLandProperty(district.getLandProperty()); + districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea())); + districtInfoDto.setConstructionSpace(district.getBuildArea()); + districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent())); + districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent())); + districtInfoDto.setDeveloper(district.getDeveloper()); + districtInfoDto.setPropertyManagement(district.getEstateCompany()); + + Integer finishYear = districtService.getCreateYear(district); + if (finishYear != null) { + districtInfoDto.setFinishYear(finishYear); + } + + mav.setModel(BasicModel.buildOKModel().setResult(districtInfoDto)); + return mav; + } + mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mav; + } + private void saveAddressMatchLog(Map resMap, String keywords, String city, String token) { Map tokenMap = FangjiaApiUtil.parseToken(token); String userName = null; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java index d938d3a8b..6fbca5e08 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java @@ -100,6 +100,7 @@ public class DistrictInfoDto { private Integer buildingCount; private Integer finishYear; private String township; + private String landProperty; public String getDistrict() { @@ -827,4 +828,12 @@ public class DistrictInfoDto { public void setTownship(String township) { this.township = township; } + + public String getLandProperty() { + return landProperty; + } + + public void setLandProperty(String landProperty) { + this.landProperty = landProperty; + } } From 6120df82cb85a491f498d39db72f29916a601c2e Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 16 May 2023 15:34:33 +0800 Subject: [PATCH 07/50] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BE=8E=E6=9C=AF?= =?UTF-8?q?=E5=AE=9D=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/MeishubaoController.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/controllers/MeishubaoController.java diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/MeishubaoController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/MeishubaoController.java new file mode 100644 index 000000000..20ea36e9c --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/MeishubaoController.java @@ -0,0 +1,61 @@ +package com.fangjia.open.controllers; + +import com.fangjia.core.util.HttpRequest; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; +import com.fangjia.open.model.AddesssMatchModel; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +@Authority(role = Role.USER) +public class MeishubaoController { + + /** + * 详细地址标准化接口 + * + * @param model + * @return + */ + public Object financialRatingAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", model.getCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/meishubao/financialRating", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/meishubao/financialRating", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + +} From 8c8543535a536e5029ba6d50275629d9509cbd29 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 16 May 2023 18:45:01 +0800 Subject: [PATCH 08/50] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkoff/controllers/DataController.java | 25 ++++++++++++++++--- .../src/main/webapp/district/add.ftl | 7 +++--- .../src/main/webapp/district/edit.ftl | 8 +++--- .../dal/mongo/model/bkoff/CityInfo.java | 10 ++++++++ 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java index bef3b773e..e0abb6dd5 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java @@ -769,6 +769,7 @@ public class DataController { outfields.add("districtsHasLoudong"); outfields.add("districtsHasFanghao"); outfields.add("priceCoverage"); + outfields.add("districtAliasTotalCount"); outfields.add("priceCreditGte80Coverage"); outfields.add("priceCreditGte66Coverage"); outfields.add("priceCreditGte60Coverage"); @@ -801,6 +802,7 @@ public class DataController { map.put("districtsHasLoudong", cityInfo.getDistrictsHasLoudong() == null ? 0 : cityInfo.getDistrictsHasLoudong()); map.put("districtsHasFanghao", cityInfo.getDistrictsHasFanghao() == null ? 0 : cityInfo.getDistrictsHasFanghao()); map.put("priceCoverage", cityInfo.getPriceCoverage()); + map.put("districtAliasTotalCount", cityInfo.getDistrictAliasTotalCount) map.put("priceCreditGte80Coverage", cityInfo.getPriceCreditGte80Coverage()); map.put("priceCreditGte66Coverage", cityInfo.getPriceCreditGte66Coverage()); map.put("priceCreditGte60Coverage", cityInfo.getPriceCreditGte60Coverage()); @@ -921,7 +923,8 @@ public class DataController { put( "estateCompany" , "物业公司" ); put( "cubagePercent" , "容积率" ); put( "virescencePercent" , "绿化率" ); - put( "carSite" , "停车位" ); + put( "carSite" , "停车位" ); + put( "carSiteCount" , "停车位描述" ); put( "estateCharge" , "物业费" ); put( "floorplanPics" , "楼层平面图" ); put( "outPics" , "外景图" ); @@ -949,7 +952,6 @@ public class DataController { put( "liveDate","开业时间"); put( "divisible","是否可分割"); put( "unifiedManage","是否统一管理"); - put( "carSiteCount","车位数量"); put( "metroInfo","地铁沿线"); put( "metroNames","地铁站名"); put( "facilities","配套设施"); @@ -996,7 +998,24 @@ public class DataController { put("committee","居委会"); put("schools","对口学校"); put("houseStructure","房屋结构"); - } };// + put("estateType2","物业类型"); + put("buildDate","建筑年代"); + put("alias","别名"); + put("addrAlias","地址别名"); + put("mainTypeArea","主力面积"); + put("loudongCount","楼栋数"); + put("township","乡镇街道"); + put("schools","对口小学"); + put("kindergartens","对口幼儿园"); + put("elevator","电梯"); + put("landProperty","权属类别"); + put("usageCard","土地使用权证号"); + put("landUsageStart","土地使用年限"); + put("occupancyRate","入住率"); + put("mainType","主力户型"); + put("mappingAlias","映射别名"); + put("districtAliasTotalCount", "别名总量"); + } };// return map; } diff --git a/projects/fangjia-bkoff/src/main/webapp/district/add.ftl b/projects/fangjia-bkoff/src/main/webapp/district/add.ftl index e7c1e520b..1a3f7a7cd 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/add.ftl @@ -257,10 +257,9 @@ checked> 灏忛珮灞�   checked> 楂樺眰   checked> 瓒呴珮灞�   - checked> 鐙爧   - checked> 鍙屾嫾   - checked> 鑱旀帓   - checked> 鍙犲姞   + checked> SOHO   + checked> LOFT   + checked> 閰掑簵寮忓叕瀵�   diff --git a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl index b856169b2..9c9421494 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl @@ -314,15 +314,15 @@ checked> 鍙犲姞   checked> 鑰佸紡鍒娲嬫埧   +
checked > 澶氬眰   checked> 灏忛珮灞�   checked> 楂樺眰   checked> 瓒呴珮灞�   - checked> 鐙爧   - checked> 鍙屾嫾   - checked> 鑱旀帓   - checked> 鍙犲姞   + checked> SOHO + checked> LOFT   + checked> 閰掑簵寮忓叕瀵�  
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/CityInfo.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/CityInfo.java index 235e3a2ee..0270b441c 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/CityInfo.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/CityInfo.java @@ -51,6 +51,7 @@ public class CityInfo implements MongoEntity { private Double priceCreditGte66Coverage; private Double priceCreditGte60Coverage; private Double priceCreditGte00Coverage; + private Integer districtAliasTotalCount; @Id @GeneratedValue @@ -378,4 +379,13 @@ public class CityInfo implements MongoEntity { public void setPriceCreditGte00Coverage(Double priceCreditGte00Coverage) { this.priceCreditGte00Coverage = priceCreditGte00Coverage; } + + @Column(name = "district_alias_total_count") + public Integer getDistrictAliasTotalCount() { + return districtAliasTotalCount; + } + + public void setDistrictAliasTotalCount(Integer districtAliasTotalCount) { + this.districtAliasTotalCount = districtAliasTotalCount; + } } From d9da00d9f922a95ee3862de7d295f5a7bfb26434 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 17 May 2023 10:09:12 +0800 Subject: [PATCH 09/50] update bug --- .../main/java/com/fangjia/bkoff/controllers/DataController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java index e0abb6dd5..26dfcdb1f 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java @@ -802,7 +802,7 @@ public class DataController { map.put("districtsHasLoudong", cityInfo.getDistrictsHasLoudong() == null ? 0 : cityInfo.getDistrictsHasLoudong()); map.put("districtsHasFanghao", cityInfo.getDistrictsHasFanghao() == null ? 0 : cityInfo.getDistrictsHasFanghao()); map.put("priceCoverage", cityInfo.getPriceCoverage()); - map.put("districtAliasTotalCount", cityInfo.getDistrictAliasTotalCount) + map.put("districtAliasTotalCount", cityInfo.getDistrictAliasTotalCount()); map.put("priceCreditGte80Coverage", cityInfo.getPriceCreditGte80Coverage()); map.put("priceCreditGte66Coverage", cityInfo.getPriceCreditGte66Coverage()); map.put("priceCreditGte60Coverage", cityInfo.getPriceCreditGte60Coverage()); From cd4029bbb25fdade71352882f0eee7d9ea477089 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 17 May 2023 10:53:07 +0800 Subject: [PATCH 10/50] =?UTF-8?q?=E5=AD=97=E6=AE=B5=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E7=8E=87=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/AddSzDealData.groovy | 3 ++ .../script/dataAna/CityInfoAnaBatch.groovy | 47 ++++++++++++++----- .../fangjia/script/gj/ccb/FileUtils.groovy | 12 +++++ .../gj/district/DistrictFiledStat.groovy | 2 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 22 ++++----- .../gj/paimai/LianjiaCrawlByChrome.groovy | 37 ++++++++++----- 6 files changed, 85 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 0a601a6ec..962d75c3b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -6,6 +6,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +/** + * 深圳公众号成交数据录入 + */ def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 45da16ec1..7447a6712 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.dataAna +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.MongoUtils import java.text.DecimalFormat @@ -56,14 +58,16 @@ class CityInfoAnaBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) final static def CITYS = []; - + + //floorplan_pics :'floorplanPics',out_pics :'outPics',out_pics : 'outPics', pics : 'pics',licence :'licence', public static final def DISTRICT_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', - all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', - car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', - start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', - alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount' + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' ] public static final def OFFICE_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', @@ -109,6 +113,7 @@ class CityInfoAnaBatch extends BaseBatch { @Override public void process(Object args) { + // 当期周末 Date weekend = DateUtil.getFirstDayOfWeek(new Date()); //周日开始跑,需要跑加上7天,周一跑不需要 @@ -123,7 +128,7 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> -// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){ +// if(["南京"].contains(k.toString())){ CITYS << k.toString(); // } } @@ -574,7 +579,7 @@ class CityInfoAnaBatch extends BaseBatch { entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4)) } entity.put('district_detail', districtDetail(disCount, cityName)); - + entity.put('district_alias_total_count', districtAliasStat(cityName)); entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName)); } else { entity.put('district_detail', [:]); @@ -1031,22 +1036,40 @@ class CityInfoAnaBatch extends BaseBatch { query.put("weekend", lastWeekend); return CITY_INFO_CONN.findOne(query); } + + def districtAliasStat(def cityName){ + DBObject query = new BasicDBObject(); + query.put('city', cityName); + query.put('cat', 'district'); + query.put('visible', 0); + DBObject fields = new BasicDBObject() + fields.put("alias", 1) + DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) + Iterator iter = cur.iterator() + def aliasCount = 0 + while(iter.hasNext()) { + def dbo = iter.next() + if(dbo && dbo.alias){ + aliasCount = aliasCount + dbo.alias.size() + } + } + return aliasCount; + } def districtDetail(def disCount, def cityName) { DBObject query = new BasicDBObject(); query.put('city', cityName); query.put('cat', 'district'); - query.put('visible', 0); + query.put('status', 0); DBObject fields = new BasicDBObject() DISTRICT_FIELDS_MAP.each() {key,skey-> fields.put(key, 1) } - + DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) Iterator iter = cur.iterator() def countMap = [:] - while(iter.hasNext()) { def dbo = iter.next() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 23513b80b..30a4e51ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -78,4 +78,16 @@ class FileUtils { } out.close(); } + + static def readTxt(def file){ + def f = new File(file) + def ret = []; + if(f.exists()){ + def lines = org.apache.commons.io.FileUtils.readLines(f) + for(line in lines) { + ret << line; + } + } + return ret; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy index 47f1563d0..e3d7eb51c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy @@ -44,7 +44,7 @@ for(def c : citys){ def v1 = weekendDetailMap.get(key) def v2 = lastWeekendDetailMap.get(key) if(v1 && v2){ - ret << DataHandleUtils.getDoubleRound(2, v1-v2) + ret << DataHandleUtils.getDoubleRound(4, v1-v2)*100 }else{ ret << "-" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ca92bb2c3..f71b47a56 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome { // // } // citys = citys[(start+1)..(citys.size()-1)] - driver = getLogin() + driver = getLogin(driver) crawlData(driver, citys) @@ -112,8 +112,8 @@ public class BeiKeCrawlByChrome { def s = driver.getPageSource() def flag = true while(flag){ - if(s && s.contains("无法访问此网站")){ - driver = getLogin() + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver) driver.get(url) Thread.sleep(5000) s = driver.getPageSource() @@ -189,13 +189,9 @@ public class BeiKeCrawlByChrome { def detailUrls = [] while(!stop){ def url = cityUrl + "pg${page}/" - try{ - driver.get(url) - }catch(Exception e){ - println url +" 出错" - } - - def doc = Jsoup.parse(driver.getPageSource()); + driver.get(url) +// def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, url); def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -426,7 +422,7 @@ public class BeiKeCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc) def list = [] - def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"] + def citys = ["福州"] if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") @@ -438,8 +434,8 @@ public class BeiKeCrawlByChrome { return list } - def getLogin(){ - WebDriver driver = new ChromeDriver() + def getLogin(driver){ +// WebDriver driver = new ChromeDriver() driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index bc5448c6c..a2e8be163 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils @@ -122,10 +123,8 @@ public class LianjiaCrawlByChrome { WebDriver driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) // cityMap = cityMap.reverse() - driver = getLogin() + driver = getLogin(driver, "https://sh.lianjia.com/") crawlData(driver, cityMap) - - } def standardRegion(){ @@ -188,8 +187,14 @@ public class LianjiaCrawlByChrome { def crawlData(driver, cityList){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt") + for(def ret : cityList){ def city = ret[0] + if(runedCitys.contains(city)){ + println city +" 已运行" + continue + } def cityUrl = ret[1] def newUrl = cityUrl+"chengjiao/" driver.get(newUrl) @@ -209,6 +214,8 @@ public class LianjiaCrawlByChrome { } } // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + + FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\链家已抓城市.txt", [], "utf-8", true); } } @@ -246,8 +253,8 @@ public class LianjiaCrawlByChrome { def s = driver.getPageSource() def flag = true while(flag){ - if(s && s.contains("无法访问此网站")){ - driver = getLogin() + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver, url) driver.get(url) Thread.sleep(5000) s = driver.getPageSource() @@ -265,7 +272,8 @@ public class LianjiaCrawlByChrome { while(!stop){ def url = cityUrl + "pg${page}/" driver.get(url) - def doc = Jsoup.parse(driver.getPageSource()); + //def detailUrls = getPageUrls(driver, cityUrl) + def doc = checkDocument(driver, url); def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -495,7 +503,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityList = [] - def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") @@ -507,13 +515,18 @@ public class LianjiaCrawlByChrome { return cityList } - def getLogin(){ - WebDriver driver = new ChromeDriver() - driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); + def getLogin(driver, url){ +// WebDriver driver = new ChromeDriver() + def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4) + driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) - wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); - + try{ + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + }catch(Exception e){ + Thread.sleep(60000) + getLogin(driver, url) + } WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 From 85b4123a232be7a3d8e7d8f1ac7377fa639c69fe Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 17 May 2023 17:42:37 +0800 Subject: [PATCH 11/50] =?UTF-8?q?=E5=86=99=E5=AD=97=E6=A5=BC=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/District2Controller.java | 3 + .../src/main/webapp/data/cityShow.ftl | 28 +- .../src/main/webapp/district2/editOffice.ftl | 785 ++++++++++-------- .../fangjia/dal/seaweed/model/District.java | 21 + 4 files changed, 454 insertions(+), 383 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java index 3d3ca01eb..4afbfe159 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java @@ -811,6 +811,9 @@ public class District2Controller { data.put("landCarSite", district.getLandCarSite()); data.put("undergroundCarSite", district.getUndergroundCarSite()); + data.put("landFloor", district.getLandFloor()); + data.put("undergroundFloor", district.getUndergroundFloor()); + data.put("clearHeight", district.getClearHeight()); data.put("standardHeight", district.getStandardHeight()); data.put("standardArea", district.getStandardArea()); diff --git a/projects/fangjia-bkoff/src/main/webapp/data/cityShow.ftl b/projects/fangjia-bkoff/src/main/webapp/data/cityShow.ftl index 701f96360..b46fd9431 100644 --- a/projects/fangjia-bkoff/src/main/webapp/data/cityShow.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/data/cityShow.ftl @@ -76,8 +76,8 @@ 鍏ヤ綇鏃堕棿 <#if disDetail.openDate??>${disDetail.openDate * 100}%<#else>鏈煡 - 鍏ヤ綇鐜� -   + 鍒悕 + <#if disDetail.alias??>${disDetail.alias * 100}%<#else>鏈煡 缁忕含搴�-mapabc <#if disDetail.latLng??>${disDetail.latLng * 100}%<#else>鏈煡 缁忕含搴�-鐧惧害 @@ -106,12 +106,12 @@ 鐗╀笟璐� <#if disDetail.estateCharge??>${disDetail.estateCharge * 100}%<#else>鏈煡 - 澶栨櫙鍥� - <#if disDetail.outPics??>${disDetail.outPics * 100}%<#else>鏈煡 - - -   -   + 绔e伐鏃堕棿 + <#if disDetail.finishBuildingDate??>${disDetail.finishBuildingDate * 100}%<#else>鏈煡 + 寤虹瓚骞翠唬 + <#if disDetail.buildDate??>${disDetail.buildDate * 100}%<#else>鏈煡 + 涓诲姏闈㈢Н + <#if disDetail.mainTypeArea??>${disDetail.mainTypeArea * 100}%<#else>鏈煡 @@ -285,7 +285,7 @@ $(document).ready(function(){ '绉熼噾', '鎴锋暟', '鍏ヤ綇鏃堕棿', - '鍏ヤ綇鐜�', + '鍒悕', '缁忕含搴�-mapabc', '缁忕含搴�-鐧惧害', '鍗犲湴闈㈢Н', @@ -297,8 +297,8 @@ $(document).ready(function(){ '缁垮寲鐜�', '鍋滆溅浣�', '鐗╀笟璐�', - '鎴垮瀷鍥�', - '澶栨櫙鍥�' + '绔e伐鏃堕棿', + '涓诲姏闈㈢Н' ], labels: { rotation: -45, @@ -328,14 +328,14 @@ $(document).ready(function(){ <#if disDetail.block??>${disDetail.block * 100}<#else>0, <#if disDetail.address??>${disDetail.address * 100}<#else>0, <#if disDetail.estateType2??>${disDetail.estateType2 * 100}<#else>0, <#if cityInfo.priceCoverage??>${cityInfo.priceCoverage * 100}<#else>0, <#if disDetail.rentPrice??>${disDetail.rentPrice * 100}<#else>0, <#if disDetail.householdCount??>${disDetail.householdCount * 100}<#else>0, - <#if disDetail.openDate??>${disDetail.openDate * 100}<#else>0, 0, + <#if disDetail.openDate??>${disDetail.openDate * 100}<#else>0, <#if disDetail.alias??>${disDetail.alias * 100}<#else>0, <#if disDetail.latLng??>${disDetail.latLng * 100}<#else>0, <#if disDetail.latLng2??>${disDetail.latLng2 * 100}<#else>0, <#if disDetail.allArea??>${disDetail.allArea * 100}<#else>0, <#if disDetail.buildArea??>${disDetail.buildArea * 100}<#else>0, <#if disDetail.buildType??>${disDetail.buildType * 100}<#else>0, <#if disDetail.developer??>${disDetail.developer * 100}<#else>0, <#if disDetail.estateCompany??>${disDetail.estateCompany * 100}<#else>0, <#if disDetail.cubagePercent??>${disDetail.cubagePercent * 100}<#else>0, <#if disDetail.virescencePercent??>${disDetail.virescencePercent * 100}<#else>0, <#if disDetail.carSite??>${disDetail.carSite * 100}<#else>0, - <#if disDetail.estateCharge??>${disDetail.estateCharge * 100}<#else>0, <#if disDetail.floorplanPics??>${disDetail.floorplanPics * 100}<#else>0, - <#if disDetail.outPics??>${disDetail.outPics * 100}<#else>0], + <#if disDetail.estateCharge??>${disDetail.estateCharge * 100}<#else>0, <#if disDetail.finishBuildingDate??>${disDetail.finishBuildingDate * 100}<#else>0, + <#if disDetail.mainTypeArea??>${disDetail.mainTypeArea * 100}<#else>0], dataLabels: { enabled: true, rotation: -90, diff --git a/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl b/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl index 8dd9ca8ee..bfb1302a3 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl @@ -258,6 +258,19 @@ + + + <#if markMap?? && markMap["township"]?? && markMap["township"]?is_number && markMap["township"] == 1> + + <#else> + + + 涔¢晣琛楅亾 + + + + + <#if markMap?? && markMap["loop_line_info"]?? && markMap["loop_line_info"] == 1> @@ -281,48 +294,48 @@ - - - - - - - - - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - - - - - - - - - - - - - - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> + + + + + + + + + + @@ -673,77 +716,81 @@ 鏄惁鍙敞鍐� - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- -->
- <#if markMap?? && markMap["intelligence_level"]?? && markMap["intelligence_level"] == 1> - - <#else> - - - 鏅鸿兘姘村钩 - -
- <#if markMap?? && markMap["licence"]?? && markMap["licence"] == 1> - - <#else> - - - 棰勫敭璁稿彲璇� - -
- <#if markMap?? && markMap["characteristic"]?? && markMap["characteristic"] == 1> - - <#else> - - - 鐗硅壊椤圭洰 - -
--> +<#-- <#if markMap?? && markMap["intelligence_level"]?? && markMap["intelligence_level"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鏅鸿兘姘村钩--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["licence"]?? && markMap["licence"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 棰勫敭璁稿彲璇�--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["characteristic"]?? && markMap["characteristic"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鐗硅壊椤圭洰--> +<#-- --> +<#--
<#if markMap?? && markMap["total_floor"]?? && markMap["total_floor"] == 1> @@ -336,31 +349,31 @@
- <#if markMap?? && markMap["meeting_room"]?? && markMap["meeting_room"] == 1> - - <#else> - - - 浼氳瀹� - -
--> +<#-- <#if markMap?? && markMap["meeting_room"]?? && markMap["meeting_room"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 浼氳瀹�--> +<#-- --> +<#--
- <#if markMap?? && markMap["live_date"]?? && markMap["live_date"] == 1> - - <#else> - - - 鍏ヤ綇鏃堕棿 - -
--> +<#-- <#if markMap?? && markMap["live_date"]?? && markMap["live_date"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鍏ヤ綇鏃堕棿--> +<#-- --> +<#--
<#if markMap?? && markMap["divisible"]?? && markMap["divisible"] == 1> @@ -389,91 +402,91 @@
- <#if markMap?? && markMap["delivery_level"]?? && markMap["delivery_level"] == 1> - - <#else> - - - 浜ゆ埧鏍囧噯 - -
- <#if markMap?? && markMap["build_facilities"]?? && markMap["build_facilities"] == 1> - - <#else> - - - 妤煎唴閰嶅 - -
--> +<#-- <#if markMap?? && markMap["delivery_level"]?? && markMap["delivery_level"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 浜ゆ埧鏍囧噯--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["build_facilities"]?? && markMap["build_facilities"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 妤煎唴閰嶅--> +<#-- --> +<#--
- <#if markMap?? && markMap["build_designer"]?? && markMap["build_designer"] == 1> - - <#else> - - - 璁捐鍗曚綅 - -
- <#if markMap?? && markMap["cons_company"]?? && markMap["cons_company"] == 1> - - <#else> - - - 鏂藉伐鍗曚綅 - -
- <#if markMap?? && markMap["rent_type"]?? && markMap["rent_type"] == 1> - - <#else> - - - 绉熷敭绫诲瀷 - -
- <#if markMap?? && markMap["lease_term"]?? && markMap["lease_term"] == 1> - - <#else> - - - 绉熻祦骞撮檺 - -
- <#if markMap?? && markMap["payment_pattern"]?? && markMap["payment_pattern"] == 1> - - <#else> - - - 浠樻鏂瑰紡 - -
--> +<#-- <#if markMap?? && markMap["build_designer"]?? && markMap["build_designer"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 璁捐鍗曚綅--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["cons_company"]?? && markMap["cons_company"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鏂藉伐鍗曚綅--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["rent_type"]?? && markMap["rent_type"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 绉熷敭绫诲瀷--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["lease_term"]?? && markMap["lease_term"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 绉熻祦骞撮檺--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["payment_pattern"]?? && markMap["payment_pattern"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 浠樻鏂瑰紡--> +<#-- --> +<#--
<#if markMap?? && markMap["estate_charge"]?? && markMap["estate_charge"] == 1> @@ -536,6 +549,32 @@ 涓�
+ <#if markMap?? && markMap["land_floor"]?? && markMap["land_floor"] == 1> + + <#else> + + + 鍦颁笂妤煎眰 + 灞� +
+ <#if markMap?? && markMap["underground_floor"]?? && markMap["underground_floor"] == 1> + + <#else> + + + 鍦颁笅妤煎眰 + 灞� +
<#if markMap?? && markMap["parking_fee"]?? && markMap["parking_fee"] == 1> @@ -661,7 +700,11 @@ 鏄惁娑夊 - +   + checked /> +        +   + checked/>
- +   + checked /> +        +   + checked/>
- <#if markMap?? && markMap["is_work_bus"]?? && markMap["is_work_bus"] == 1> - - <#else> - - - 鏄惁鏈夌彮杞� - -
--> +<#-- <#if markMap?? && markMap["is_work_bus"]?? && markMap["is_work_bus"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鏄惁鏈夌彮杞�--> +<#-- --> +<#--
- <#if markMap?? && markMap["work_bus_time"]?? && markMap["work_bus_time"] == 1> - - <#else> - - - 鐝溅鏃堕棿 - -
--> +<#-- <#if markMap?? && markMap["work_bus_time"]?? && markMap["work_bus_time"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鐝溅鏃堕棿--> +<#-- --> +<#--
- <#if markMap?? && markMap["lighting_ventilation"]?? && markMap["lighting_ventilation"] == 1> - - <#else> - - - 绌鸿皟 - -
- <#if markMap?? && markMap["conditioner_time"]?? && markMap["conditioner_time"] == 1> - - <#else> - - - 绌鸿皟寮�鏀炬椂闂� - -
--> +<#-- <#if markMap?? && markMap["lighting_ventilation"]?? && markMap["lighting_ventilation"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 绌鸿皟--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["conditioner_time"]?? && markMap["conditioner_time"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 绌鸿皟寮�鏀炬椂闂�--> +<#-- --> +<#--
- <#if markMap?? && markMap["conditioner_timelag"]?? && markMap["conditioner_timelag"] == 1> - - <#else> - - - 绌鸿皟鍔犳椂 - -
--> +<#-- <#if markMap?? && markMap["conditioner_timelag"]?? && markMap["conditioner_timelag"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 绌鸿皟鍔犳椂--> +<#-- --> +<#--
@@ -757,18 +804,18 @@ 涓�
- <#if markMap?? && markMap["elevator_brand"]?? && markMap["elevator_brand"] == 1> - - <#else> - - - 鐢垫鍝佺墝 - -
--> +<#-- <#if markMap?? && markMap["elevator_brand"]?? && markMap["elevator_brand"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鐢垫鍝佺墝--> +<#-- --> +<#--
<#if markMap?? && markMap["person_elevators_count"]?? && markMap["person_elevators_count"] == 1> @@ -804,18 +851,18 @@
- - - - +<#-- --> +<#-- --> +<#-- --> +<#-- -->
- <#if markMap?? && markMap["fitment"]?? && markMap["fitment"] == 1> - - <#else> - - - 瑁呬慨鎯呭喌 - -
--> +<#-- <#if markMap?? && markMap["fitment"]?? && markMap["fitment"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 瑁呬慨鎯呭喌--> +<#-- --> +<#--
<#if markMap?? && markMap["open_date"]?? && markMap["open_date"] == 1> @@ -922,140 +969,140 @@
- - - - - - - - - - - -
- <#if markMap?? && markMap["safeSystem"]?? && markMap["safeSystem"] == 1> - - <#else> - - - 瀹夐槻绯荤粺 - -
- <#if markMap?? && markMap["network_way"]?? && markMap["network_way"] == 1> - - <#else> - - - 缃戠粶閫氳 - -
+<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["safeSystem"]?? && markMap["safeSystem"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 瀹夐槻绯荤粺--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["network_way"]?? && markMap["network_way"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 缃戠粶閫氳--> +<#-- --> +<#--
--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#--
--> +<#-- --> +<#--
--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["metro_info"]?? && markMap["metro_info"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鍦伴搧娌跨嚎--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["fitment_level"]?? && markMap["fitment_level"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 瑁呬慨鏍囧噯--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["metro_names"]?? && markMap["metro_names"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鍦伴搧绔欏悕--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["surrounding"]?? && markMap["surrounding"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鍛ㄨ竟閰嶅--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["facilities"]?? && markMap["facilities"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 閰嶅璁炬柦--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["traffic"]?? && markMap["traffic"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 浜ら�氭鍐�--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["property_info"]?? && markMap["property_info"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鐗╀笟姒傚喌--> +<#-- --> +<#--
--> - - - - -
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- <#if markMap?? && markMap["metro_info"]?? && markMap["metro_info"] == 1> - - <#else> - - - 鍦伴搧娌跨嚎 - -
- <#if markMap?? && markMap["fitment_level"]?? && markMap["fitment_level"] == 1> - - <#else> - - - 瑁呬慨鏍囧噯 - -
- <#if markMap?? && markMap["metro_names"]?? && markMap["metro_names"] == 1> - - <#else> - - - 鍦伴搧绔欏悕 - -
- <#if markMap?? && markMap["surrounding"]?? && markMap["surrounding"] == 1> - - <#else> - - - 鍛ㄨ竟閰嶅 - -
- <#if markMap?? && markMap["facilities"]?? && markMap["facilities"] == 1> - - <#else> - - - 閰嶅璁炬柦 - -
- <#if markMap?? && markMap["traffic"]?? && markMap["traffic"] == 1> - - <#else> - - - 浜ら�氭鍐� - -
- <#if markMap?? && markMap["property_info"]?? && markMap["property_info"] == 1> - - <#else> - - - 鐗╀笟姒傚喌 - -
- - - - - -
- -
+<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#--
--> +<#-- --> +<#--
--> diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index e7c2195ef..2bb00ca36 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -159,6 +159,9 @@ public class District implements SeaweedEntity { private String landCarSite;// 地上车位数 private String undergroundCarSite;// 地下车位数 + private String landFloor;// 地上层数 + private String undergroundFloor;// 地下层数 + private String clearHeight;// 净高 private String standardHeight;// 标准层高 private String standardArea;// 标准层面积 @@ -1873,4 +1876,22 @@ public class District implements SeaweedEntity { public void setNameCat(Integer nameCat) { this.nameCat = nameCat; } + + @Column(name = "land_floor") + public String getLandFloor() { + return landFloor; + } + + public void setLandFloor(String landFloor) { + this.landFloor = landFloor; + } + + @Column(name = "underground_floor") + public String getUndergroundFloor() { + return undergroundFloor; + } + + public void setUndergroundFloor(String undergroundFloor) { + this.undergroundFloor = undergroundFloor; + } } From c6ebf48a81d9030b9b00c36144a0ea7d1a3f1bc1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 18 May 2023 11:08:06 +0800 Subject: [PATCH 12/50] =?UTF-8?q?=E5=85=AC=E4=BC=97=E5=8F=B7=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/WeixinController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/WeixinController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/WeixinController.java index 8d66e0182..b54ec1dad 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/WeixinController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/WeixinController.java @@ -388,7 +388,8 @@ public class WeixinController { return postImageMessage.format(); }else{ PostTextMessage postMessage = new PostTextMessage( - "亲爱的用户,感谢关注房价网官方微信"+"\n" +"1.发送“PDS”,申请试用PDS不动产大数据服务平台;"+"\n" +"2.发送小区名称,即可获得小区均价及详情;"+"\n"+"3.发送城市名称,即可获得城市二手房均价;"+"\n"+"4.发送“调查”,即可参与小区调查;"+"\n"+"更多内容请登陆www.fangjia.com"); + "欢迎关注房价网!"+"\n" +"想了解我们的产品服务及更多企业级地址数据解决方案,可登录官网http://sh.fangjia.com/\n" + + "一对一在线人工咨询,可添加企业客服微信13816692251"+"\n"+"全国客服热线:400-188-3737"+"\n\n"+"市场品牌或渠道合作,发邮件至gujia@fangjia.com"); postMessage.setToUserName(receivedMessage.getFromUserName()); return postMessage.format(); } From c5e637ee10117350d7d404941f9ee738358a86ae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 18 May 2023 12:42:24 +0800 Subject: [PATCH 13/50] test --- .../fangjia/open/controllers/weixin/DistrictController.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java index 826c55070..674834e68 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java @@ -141,6 +141,7 @@ public class DistrictController { } public Object districtDetailAction(DistrictModel districtModel,HttpServletRequest request,HttpServletResponse response) { + System.out.println("小区详情coming"); //id和name不能同时为空 if(StringUtils.isBlank(districtModel.getId()) && (StringUtils.isBlank(districtModel.getCityName()) || StringUtils.isBlank(districtModel.getName()))) { districtModel.setMsg("参数错误"); @@ -149,13 +150,16 @@ public class DistrictController { District district = null; if(StringUtils.isNotBlank(districtModel.getId())) { + System.out.println("小区详情1"); district = districtService.findById(districtModel.getId()); + if(district != null) { districtModel.setCityName(district.getCity()); districtModel.setRegion(district.getRegion()); districtModel.setName(district.getName()); } } else { + System.out.println("小区详情2"); district = districtService.findByKeyMiniModel(districtModel.getCityName(), districtModel.getRegion(), districtModel.getName(), "district"); } From 33687df49954e7059bc13ca2617d0c8cbe367246 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 18 May 2023 18:21:41 +0800 Subject: [PATCH 14/50] =?UTF-8?q?=E5=86=99=E5=AD=97=E6=A5=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/District2Controller.java | 1 + .../src/main/webapp/district2/addOffice.ftl | 393 ++++++++++-------- 2 files changed, 212 insertions(+), 182 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java index 4afbfe159..52a578dbd 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java @@ -756,6 +756,7 @@ public class District2Controller { data.put("carSiteCount", district.getCarSiteCount()); data.put("modificationDate", new Date()); data.put("publisherType", 11); + data.put("township", district.getTownship()); data.put("businessQuarter", district.getBusinessQuarter()); diff --git a/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl b/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl index 207cd11ae..6e6b47bc5 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl @@ -156,36 +156,43 @@ + + 涔¢晣琛楅亾 + + + + + 鐜嚎浣嶇疆 - - 鏅鸿兘姘村钩 - - - - - - 棰勫敭璁稿彲璇� - - - - - - 鐗硅壊椤圭洰 - - - - +<#-- --> +<#-- 鏅鸿兘姘村钩--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 棰勫敭璁稿彲璇�--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 鐗硅壊椤圭洰--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> 鎬绘ゼ灞� @@ -194,19 +201,19 @@ - - 浼氳瀹� - - - - +<#-- --> +<#-- 浼氳瀹�--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - 鍏ヤ綇鏃堕棿 - - - - +<#-- --> +<#-- 鍏ヤ綇鏃堕棿--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> 鏄惁鍙垎鍓� @@ -223,49 +230,49 @@ - - 浜ゆ埧鏍囧噯 - - - - - - 妤煎唴閰嶅 - - - - +<#-- --> +<#-- 浜ゆ埧鏍囧噯--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 妤煎唴閰嶅--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - 璁捐鍗曚綅 - - - - - - 鏂藉伐鍗曚綅 - - - - - - 绉熷敭绫诲瀷 - - - - - - 绉熻祦骞撮檺 - - - - - - 浠樻鏂瑰紡 - - - - +<#-- --> +<#-- 璁捐鍗曚綅--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 鏂藉伐鍗曚綅--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 绉熷敭绫诲瀷--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 绉熻祦骞撮檺--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 浠樻鏂瑰紡--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> 鐗� 涓� 璐� @@ -297,6 +304,19 @@ + + + 鍦颁笂灞傛暟 + + 灞� + + + + 鍦颁笅灞傛暟 + + 灞� + + 杞︿綅鏈堢閲� @@ -346,18 +366,27 @@ 绫� - + + 鏄惁娑夊 - - - + +   + +        +   + + 鏄惁鍙敞鍐� - - - + +   + +        +   + + @@ -368,31 +397,31 @@ - - 鐝溅鏃堕棿 - - - - - - 绌鸿皟 - - - - - - 绌鸿皟寮�鏀炬椂闂� - - - - +<#-- --> +<#-- 鐝溅鏃堕棿--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 绌鸿皟--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 绌鸿皟寮�鏀炬椂闂�--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - 绌鸿皟鍔犳椂 - - - - +<#-- --> +<#-- 绌鸿皟鍔犳椂--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> 鐢垫鏁伴噺 @@ -400,12 +429,12 @@ 涓� - - 鐢垫鍝佺墝 - - - - +<#-- --> +<#-- 鐢垫鍝佺墝--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> 瀹㈡鏁� @@ -419,12 +448,12 @@ - - 瑁呬慨鎯呭喌 - - - - +<#-- --> +<#-- 瑁呬慨鎯呭喌--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> @@ -475,67 +504,67 @@ - - 瀹夐槻绯荤粺 - - - - - - - - 缃戠粶閫氳 - - - - - - - - 鍦伴搧娌跨嚎 - - - - - - 瑁呬慨鏍囧噯 - - - - - - 鍦伴搧绔欏悕 - - - - - - - 鍛ㄨ竟閰嶅 - - - - - - 閰嶅璁炬柦 - - - - - - - 浜ら�氭鍐� - - - - - - - 鐗╀笟姒傚喌 - - - - +<#-- --> +<#-- 瀹夐槻绯荤粺--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 缃戠粶閫氳--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 鍦伴搧娌跨嚎--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 瑁呬慨鏍囧噯--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 鍦伴搧绔欏悕--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 鍛ㄨ竟閰嶅--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 閰嶅璁炬柦--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 浜ら�氭鍐�--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- 鐗╀笟姒傚喌--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> From 7f38659cec3fcae66d6c34e5f93b7484175392e2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 27 May 2023 11:04:47 +0800 Subject: [PATCH 15/50] update --- .../script/ccbevaluation/AddSzDealData.groovy | 4 +- .../BigdataCommunityBatch.groovy | 12 +-- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../script/ccbevaluation/TestRepeat.groovy | 18 +++- .../ronghe/ExportDistrictRongHe.groovy | 83 ++++++++++--------- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../fangjia/script/gj/ccb/FileUtils.groovy | 2 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 60 ++++++-------- .../gj/paimai/LianjiaCrawlByChrome.groovy | 30 ++++--- 9 files changed, 110 insertions(+), 105 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 962d75c3b..c81c099a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\3月坪山龙华盐田罗湖龙岗成交数据.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -29,7 +29,7 @@ for(def d : datas){ map.area = d['F'].toString().toDouble() map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) map.avg_price = (int)(d['H'].toString().toDouble()*10000) - def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31") + def trade_date = DateUtils.getRandomDate("2023-04-01", "2023-04-30") map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd") map.source = "深房家族" map.create_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 345491bd5..c52b21fcb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 -// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 +// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -78,10 +78,10 @@ class BigdataCommunityBatch extends BaseBatch{ println "txt 生成->导Oracle" -// delPgData() -// saveToPgSql(rows) - // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") + delPgData() + saveToPgSql(rows) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") // delPgData() // saveToPgSql2(datas) @@ -259,7 +259,7 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(1, sql){ ps-> + sqlInstance.withBatch(50000, sql){ ps-> for(def r : rows) { try{ ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index de1e2d965..d276e8c73 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -430,7 +430,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); - q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 +// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 // q.put("name","锦南浦三小区") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy index c0833369a..b2e287b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.mongodb.BasicDBObject import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap @@ -12,15 +15,24 @@ class TestRepeat { ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + private DSCollection seabird = new DataStore("192.168.0.100", 37017).getCollection("fangjia.seabird") static main(args) { def task = new TestRepeat() task.run() } def run(){ - + def q = new BasicDBObject() + q.put("cat", "sell") + q.put("city", "襄阳") + def count = seabird.count(q) + println count +// for(def d : dbcursor){ +// println d.name +// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 43132328a..b4f5dcad1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -39,48 +39,49 @@ class ExportDistrictRongHe extends BaseBatch{ citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); +// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) - exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +//// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// +// if(curDate.toString().endsWith("10")){ +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// +// +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() +// +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) +// +// +// +// //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 +//// exportCommunityBasePrice(lastMonth) +//// exportCommunityPdescribeGj(lastMonth) +//// exportGuideStatistics(lastMonth) +//// exportBigdataSupplierStatistics(lastMonth) +//// exportBigdataModelingBase(lastMonth) +// +// //2月8号 临时导出苏州一年的数据 +// exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 +// } - if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// - - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() - - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) - - - - //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 -// exportCommunityBasePrice(lastMonth) -// exportCommunityPdescribeGj(lastMonth) -// exportGuideStatistics(lastMonth) -// exportBigdataSupplierStatistics(lastMonth) -// exportBigdataModelingBase(lastMonth) - - //2月8号 临时导出苏州一年的数据 - exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 - } - - if(curDate.toString().endsWith("28")){ + if(curDate.toString().endsWith("20")){ exportPgBigdataSupplierStatistics(lastMonth) //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); @@ -92,10 +93,10 @@ class ExportDistrictRongHe extends BaseBatch{ exportPgGuideStatistics(lastMonth) } - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - - exportOk(curDate) +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// +// exportOk(curDate) } // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 0842f9b06..16cae9383 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-03-01" + static def statDate = "2023-04-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 30a4e51ab..78270dc6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -83,7 +83,7 @@ class FileUtils { def f = new File(file) def ret = []; if(f.exists()){ - def lines = org.apache.commons.io.FileUtils.readLines(f) + def lines = org.apache.commons.io.FileUtils.readLines(f,"utf-8") for(line in lines) { ret << line; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index f71b47a56..91f635c3f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils @@ -49,27 +50,8 @@ public class BeiKeCrawlByChrome { WebDriver driver = new ChromeDriver() def citys = getAllCityUrls(driver) -// citys = citys.reverse() -// def index = 0 -// for(def c : citys){ -// index++ -// if(c.city=="合肥"){ -// break -// } -// } -// citys = citys[index..citys.size()-1] - def start = 0 -// for(def i=0;i登录"))){ - driver = getLogin(driver) + driver = getLogin(driver, url) driver.get(url) Thread.sleep(5000) s = driver.getPageSource() @@ -127,8 +109,14 @@ public class BeiKeCrawlByChrome { def crawlData(driver, citys){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt") + for(def m : citys){ def city = m[0] + if(runedCitys.contains(city)){ + println city +" 已运行" + continue + } def url = m[1] def newUrl = url+"/chengjiao/" driver.get(newUrl) @@ -145,10 +133,11 @@ public class BeiKeCrawlByChrome { println url +" 详情链接:"+detailUrls.size() } } - + FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true); // def detailUrls = getPageDeail(driver, m.city, m.url) // println m.city +" 详情链接:"+detailUrls.size() // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + } } @@ -434,27 +423,32 @@ public class BeiKeCrawlByChrome { return list } - def getLogin(driver){ + def getLogin(driver, url){ // WebDriver driver = new ChromeDriver() - driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); + def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4) + driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) - wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + try{ + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); - WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 - WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 - WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - elemUsername.sendKeys("15021384995"); + elemUsername.sendKeys("15021384995"); - elemPassword.sendKeys("12qwaszx"); + elemPassword.sendKeys("12qwaszx"); - btn.submit(); // 点击行为; - Thread.sleep(10000) + btn.submit(); // 点击行为; + Thread.sleep(10000) + }catch(Exception e){ + Thread.sleep(60000) + } return driver } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index a2e8be163..aaf0c9c0d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -118,6 +118,7 @@ public class LianjiaCrawlByChrome { } def run(){ + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() @@ -523,24 +524,21 @@ public class LianjiaCrawlByChrome { Thread.sleep(10000) try{ wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + + elemUsername.sendKeys("15021384995"); + + elemPassword.sendKeys("12qwaszx"); + + btn.submit(); // 点击行为; + Thread.sleep(10000) }catch(Exception e){ - Thread.sleep(60000) - getLogin(driver, url) + Thread.sleep(10000) } - - WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 - - WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 - - WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - - - elemUsername.sendKeys("15021384995"); - - elemPassword.sendKeys("12qwaszx"); - - btn.submit(); // 点击行为; - Thread.sleep(20000) return driver } } From f20cff4a295e12ae58205bbf9594ca484656cf7e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 27 May 2023 11:05:48 +0800 Subject: [PATCH 16/50] update --- .../script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index b4f5dcad1..a9aba9f10 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -44,8 +44,8 @@ class ExportDistrictRongHe extends BaseBatch{ def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -//// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) -// exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) // // if(curDate.toString().endsWith("10")){ // exportBigdataCommunityGJRH()// From baf6f6c5a4d6e38c12dfcd4c51a3377f7eb4b780 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 19 May 2023 13:12:26 +0800 Subject: [PATCH 17/50] update --- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ronghe/ExportDistrictRongHe.groovy | 95 +++++++++---------- 2 files changed, 47 insertions(+), 50 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index db2c71d84..bb750e5ff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230318, 20230419, houseIdsfilter); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index a9aba9f10..852f3783a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -39,64 +39,61 @@ class ExportDistrictRongHe extends BaseBatch{ citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); -// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) -// -// if(curDate.toString().endsWith("10")){ -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// -// -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() -// -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) -// -// -// -// //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 -//// exportCommunityBasePrice(lastMonth) -//// exportCommunityPdescribeGj(lastMonth) -//// exportGuideStatistics(lastMonth) -//// exportBigdataSupplierStatistics(lastMonth) -//// exportBigdataModelingBase(lastMonth) -// -// //2月8号 临时导出苏州一年的数据 -// exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 -// } - if(curDate.toString().endsWith("20")){ - exportPgBigdataSupplierStatistics(lastMonth) - //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) - exportPgThirdPartyPrice2(thirdPartyMonth, new_dists) - exportPgCommunityBasePrice(lastMonth) - exportPgCommunityPdescribeGj(lastMonth) - exportPgBigdataModelingBase(lastMonth) - exportPgGuideStatistics(lastMonth) + if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 +// exportCommunityBasePrice(lastMonth) +// exportCommunityPdescribeGj(lastMonth) +// exportGuideStatistics(lastMonth) +// exportBigdataSupplierStatistics(lastMonth) +// exportBigdataModelingBase(lastMonth) + + //2月8号 临时导出苏州一年的数据 + exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 } -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// -// exportOk(curDate) + if(curDate.toString().endsWith("28")){ + exportPgBigdataSupplierStatistics(curMonth) + //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); +// exportThirdPartyPrice2(thirdPartyMonth, new_dists) + exportPgThirdPartyPrice2(thirdPartyMonth, new_dists) + exportPgCommunityBasePrice(curMonth) + exportPgCommunityPdescribeGj(curMonth) + exportPgBigdataModelingBase(curMonth) + exportPgGuideStatistics(curMonth) + } + + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); + + exportOk(curDate) } // From b79c5aaede9f4d365f0a119d2eb2823524f0f9c4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 19 May 2023 13:32:58 +0800 Subject: [PATCH 18/50] update --- .../script/ccbevaluation/BigdataCommbuildtypeBatch.groovy | 6 ++++++ .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 1 + 2 files changed, 7 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 3418defb3..61bd39b7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -56,6 +56,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" + delPgData() saveToPgSql(rets) } @@ -110,6 +111,11 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ return rets; } + def delPgData(){ + sqlInstance.execute("delete from base.t0831_bigdata_commbuildtype_gj_h") + println "base.t0831_bigdata_commbuildtype_gj_h 数据已删除" + } + def saveToPgSql(rows){ def sqls = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index bb750e5ff..31fa93b9d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -1127,6 +1127,7 @@ class BigdataForsaleBatch extends BaseBatch{ return count; } + def saveToPgSql(rows){ for(def r : rows) { for (def i = 0; i < r.size(); i++) { From b762564565213493fd3164a8a13cf2f501215d38 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 19 May 2023 13:51:08 +0800 Subject: [PATCH 19/50] =?UTF-8?q?=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 84bbeaa2b..da8662b0e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202303//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202301 - def endDate = 202303 + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304// + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From eefd91aa7c4e1e4d9eb19611684c5f75c2bc447f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 19 May 2023 15:52:57 +0800 Subject: [PATCH 20/50] =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/AddSzDealData.groovy | 45 +++++++++++++++---- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index c81c099a9..f34d40421 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,10 +9,41 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\3月坪山龙华盐田罗湖龙岗成交数据.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) +//没有成交时间,随机生成 +//for(def d : datas){ +// def map = new BasicDBObject() +// map.city = "深圳" +// map.district_name = d['D'] +// map.region = d['B'] +// def roomStr = d['E'] +// if(roomStr){ +// def arrs = roomStr.split("-") +// try{ +// map.room = arrs[0].toString().toInteger() +// }catch(Exception e){ +// println "room error" +// } +// map.hall = arrs[1].toString().toInteger() +// map.toilet = arrs[3].toString().toInteger() +// } +// +// map.area = d['F'].toString().toDouble() +// map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) +// map.avg_price = (int)(d['H'].toString().toDouble()*10000) +// def trade_date = DateUtils.getRandomDate("2023-04-01", "2023-04-30") +// map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd") +// map.source = "深房家族" +// map.create_date = new Date() +// map.m_date = new Date() +// DEAL_PRICE_COLL.save(map) +//} + + + for(def d : datas){ def map = new BasicDBObject() map.city = "深圳" @@ -20,17 +51,15 @@ for(def d : datas){ map.region = d['B'] def roomStr = d['E'] if(roomStr){ - def arrs = roomStr.split("-") - map.room = arrs[0].toString().toInteger() - map.hall = arrs[1].toString().toInteger() - map.toilet = arrs[3].toString().toInteger() + map.room = roomStr.toString().toDouble().toInteger() } map.area = d['F'].toString().toDouble() map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) - map.avg_price = (int)(d['H'].toString().toDouble()*10000) - def trade_date = DateUtils.getRandomDate("2023-04-01", "2023-04-30") - map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd") + map.avg_price = (int)(d['H'].toString().toDouble()) + map.sell_price = (int)(d['I'].toString().toDouble()*10000) + def trade_date = d['J'] + map.trade_date = trade_date map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 31fa93b9d..f4e661538 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -203,7 +203,7 @@ class BigdataForsaleBatch extends BaseBatch{ } } - println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString() + println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size() } catch (Exception e) { e.printStackTrace() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index d276e8c73..d0a216521 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -431,7 +431,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","锦南浦三小区") +// q.put("name","金地玺湾") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) From 591b5d6c97f689d687cdba7b51530078d4052514 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 22 May 2023 11:24:16 +0800 Subject: [PATCH 21/50] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/AddressController.java | 1 + .../java/com/fangjia/open/model/AddesssMatchModel.java | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index ba0985a47..3b9563cbe 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1837,6 +1837,7 @@ public class AddressController { Map params = new HashMap(); params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); params.put("address", model.getAddress()); + params.put("showCode", model.getShowCode()); try{ json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); } catch(Exception e){ diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java index e1048387a..ea10ab2f4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java @@ -22,6 +22,8 @@ public class AddesssMatchModel extends FangjiaApiModel { private String address_1; private String address_2; + private String showCode; + public String getAddress_1() { return address_1; } @@ -134,4 +136,12 @@ public class AddesssMatchModel extends FangjiaApiModel { public void setDebugName(String debugName) { this.debugName = debugName; } + + public String getShowCode() { + return showCode; + } + + public void setShowCode(String showCode) { + this.showCode = showCode; + } } From 3a3231244e605bf40c0298bccdf1790a50eb6d26 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 22 May 2023 11:59:45 +0800 Subject: [PATCH 22/50] update --- .../ccbevaluation/BigdataDealBatch.groovy | 85 +++++++++---------- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 5e7d45516..6854ae591 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -65,52 +65,51 @@ class BigdataDealBatch extends BaseBatch{ if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } - println "curDate:"+curDate - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } - } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 // for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } // } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -// saveToPgSql(rows) + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index b78a31710..8041a896e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-26") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-20") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) //更新静态数据表 -// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } From 579e7ac368521fe99574aa7ffbd55225652bbf25 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 22 May 2023 16:20:47 +0800 Subject: [PATCH 23/50] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AF=BC=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 10 +- .../script/ccbevaluation/UpdateData.groovy | 23 +- .../pgsql/ExportAllPgData.groovy | 1020 +++++++++++++++++ .../script/dataAna/CityInfoAnaBatch.groovy | 4 +- .../com/fangjia/script/gj/FiledStrand.groovy | 5 +- .../gj/district/DistrictFieldAddBatch.groovy | 4 +- .../district/SeaweedDataMatchProcess.groovy | 6 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 3 +- 9 files changed, 1062 insertions(+), 15 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 8041a896e..953a53a6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-20") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3c0226e81..e8bd49824 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-01-01" - def date = "2023-02-01" - def endDate = "2023-03-01" - def newDate = "2023-04-01" + def startDate = "2023-02-01" + def date = "2023-03-01" + def endDate = "2023-04-01" + def newDate = "2023-05-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy index 024ad4123..640b024b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import groovy.sql.Sql @@ -11,7 +13,26 @@ class UpdateData { static main(args) { def task = new UpdateData(); - task.run() +// task.run() + task.updateBlock() + } + + def updateBlock(){ + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "block","name"]) + for(def d : dists){ + if(d.block){ + def newBlock = FiledStrand.processBlock(d.city, d.region, d.block) + if(newBlock!= d.block){ + def udata = new BasicDBObject() + udata.put("block", newBlock) + println d.city+" "+d.block +" "+newBlock + DbUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata)) + } + } + } + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy new file mode 100644 index 000000000..6e9afa65b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -0,0 +1,1020 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +class ExportAllPgData { + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "base" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportAllPgData() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def new_dists = [] + + exportEvaluationRateData() + exportBigdataPoiData() + exportBigdataDistrictData() +// +// //第三方网站权重表 + exportWebsiteRate(); + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice()//随政策// + //处理小区情况 + exportCommunityStaging()//没cityid + exportBigdataCommunity()// + + exportCommunityBuildType()// + + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //现在按updatetime导出 + exportBigdataDeal(new_dists) //2022-07-01 + + + //计算结果相关 + exportAvmEvaluationGJNB() + exportAVMEvaluationItemGJNB() + + exportCityPrice() + exportRegionPrice() + + exportCommunityBasePrice() +// exportCommunityBasePrice1(curMonth) + + exportCommunityPdescribeGj() + exportCommCaseDetailGj() +// + exportInOk() +// + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + } + + def exportEvaluationRateData() { + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_bds_ceft + data << r.gujia_bds_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + + + def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false) + } + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_fsid + data << r.poi_nm + data << r.gujia_tp + data << r.gujia_tp_ecd + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_lgc_admndiv_dsc + data << r.gujia_lgc_admndiv_cd + data << r.gujia_adiv_cd_val + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false) + } + + def exportWebsiteRate(){ + def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujiadatawebst_src_id + data << r.gujia_data_webst_src + data << r.gujiadatawebstmpng_id + data << r.gujia_bds_wgt + data << r.gujia_bds_tp + data << r.gujia_src_co + data << r.urbn_id + data << r.gujia_urbn + data << r.data_crt_tm + data << r.data_udt_tm + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false) + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_dt + data << r.gujia_bds_drc_prc + data << r.gujia_add_psn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_drc_prc + data << r.gujia_bds_drc_prc_src + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from "+database+".t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + pgsqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_bds_st + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_instm_id + data << r.gujia_comm_id + data << r.gujiacomm_prim_prd_id + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityBuildType() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_tp_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false) + } + + def exportThirdPartyPrice2(new_dists) { + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_dtsr_id + data << r.gujia_dtsr_dsc + data << r.gujiabds_avprc_dt_val + data << r.gujia_bds_comm_avprc + data << r.gujiabdscomm_hgst_prc + data << r.gujiabdscomm_lwst_prc + data << r.data_udt_tm + allRows << data + }) + return allRows; + } + + + def exportBigdataDeal(def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'"; + allRows.addAll(dealData(sql)) + } + } + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false) + } + + + def dealData(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_bds_mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_bds_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_bds_trdtckt_prc + data << r.gujia_bds_mdl_tot_prc + data << r.gujia_bds_dcl_tot_prc + data << r.gujiabds_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_bds_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiabdslndfgrngvarea + data << r.gujiabdslutbl_gv_area + data << r.gujiabdssuntblgv_area + data << r.gujiabdsbayindowgarea + data << r.gujiabdstowerfrgvarea + data << r.gujia_bds_grg_gv_area + data << r.gujiabdsundgdrmgvarea + data << r.gujiabds_othr_gv_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_mkdam + data << r.gujia_bds_data_tm_val + allRows << data + }); + return allRows + } + + + def exportBigdataForsale(new_dists){ + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' "; + allRows = getHouseDatas(sql) + } + println city +" "+allRows.size() + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + pgsqlInstance.eachRow(sql,{r-> + index = index+1 + def data = [] + data << r.urbn_id + data << r.gujia_sell_id + data << r.gujia_listg_hs_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_bds_unitprc + data << r.gujia_bds_tot_prc + data << r.gujia_sell_pymtmod + data << r.gujia_bds_wbt_floor + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_indr_stc_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujiawthrext_lift_ind + data << r.gujiajiladderjiactval + data << r.anc_tm + data << r.exdat + data << r.gujia_wthr_strt + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.gujia_anc_psn + data << r.gujia_anc_psn_tel + data << r.gujia_dsc_inf_cntnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_apd_inf + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportAvmEvaluationGJNB(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_bds_yrmo,gujia_bds_comm_num,gujiabdsvalmnycommnum,gujia_prc_cvr_rate,gujiabdsmnldpccommnum,gujia_mnl_drc_prc_pct,gujiabdssamp_comm_num,gujiaext_samp_clc_pct,gujiabdsnervetncomnum,gujianervethrunclcpct,gujiabds_tst_case_num,gujiabds_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiabdsmdltstcasenum,gujiabdsmdltstcommnum,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiabdstrdpttcasenum,gujiabdstrdpttcommnum,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_bds_yrmo + data << r.gujia_bds_comm_num + data << r.gujiabdsvalmnycommnum + data << r.gujia_prc_cvr_rate + data << r.gujiabdsmnldpccommnum + data << r.gujia_mnl_drc_prc_pct + data << r.gujiabdssamp_comm_num + data << r.gujiaext_samp_clc_pct + data << r.gujiabdsnervetncomnum + data << r.gujianervethrunclcpct + data << r.gujiabds_tst_case_num + data << r.gujiabds_tst_comm_num + data << r.gujiatst_comm_num_pct + data << r.gujia_ppe15_cvr_rate + data << r.gujia_rto_mdn + data << r.gujia_rto_lv_san_ceft + data << r.gujiabdsmdltstcasenum + data << r.gujiabdsmdltstcommnum + data << r.gujiamdltstcommnumpct + data << r.gujiamdltst_ppe15_pct + data << r.gujiamdlrtolvsan_ceft + data << r.gujia_mdl_md_bit_rto + data << r.gujiabdstrdpttcasenum + data << r.gujiabdstrdpttcommnum + data << r.gujiatrdpttcommnumpct + data << r.gujiatrdpttstppe15pct + data << r.gujiatrdptrolvsanceft + data << r.gujiatrdpt_md_bit_rto + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false) + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_avprc,gujia_bds_clc_tp,gujia_bds_rlb,gujiabds_to_mkprc_mny,to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujiabds_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_bds_floor_num,gujiabdstot_floor_num,gujia_data_tp,gujia_rto,gujia_mns_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') AS data_crt_tm from "+database+"avm_evaluation_item_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_rgon_id + data << r.gujia_rgon_nm + data << r.gujia_comm_id + data << r.gujia_comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_avprc + data << r.gujia_bds_clc_tp + data << r.gujia_bds_rlb + data << r.gujiabds_to_mkprc_mny + data << r.mdl_dt + data << r.gujiabds_valt_unitprc + data << r.gujia_area + data << r.gujia_dcrt_val + data << r.gujia_twrd_val + data << r.gujia_bds_floor_num + data << r.gujiabdstot_floor_num + data << r.gujia_data_tp + data << r.gujia_rto + data << r.gujia_mns_rto + data << r.data_crt_tm + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false) + } + + def exportCityPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportRegionPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where urbn_id='${cityCode}'"; // + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gxda_admndiv_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportCommunityBasePrice(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false) + createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false) + } + + + def exportCommunityBasePrice1(yearMonth){ + + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + } + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + } + } + + def exportCommunityPdescribeGj(){ + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from "+database+"avm_community_p_describe_gj_h where urbn_id='${cityCode}'"; + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_comm_id + data << r.urbn_id + data << r.urbn_nm + data << r.admndiv_ecd + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_als_dsc + data << r.gujia_bds_bss_prc + data << r.gujiabds_cfdc_dgr_grd + data << r.gujiabdsrt3mlgcasenum + data << r.gujiabdslistgclwstprc + data << r.gujiabdslistgcaemdval + data << r.gujiabdslistgcase_avg + data << r.gujiabdslistgcemaxprc + data << r.gujia_bds_mo_val + data << r.gujiabdsavgwminvalavg + data << r.gujiabdsavgwmaxvalavg + data << r.gujia_bds_rel_err_val + data << r.gujia_dvtdgr_dsc + data << r.gujia_als_prc_rng + data << r.gujiaprcminandmax_rng + data << r.gujia_dsc + data << r.gujiabds_lmth_bss_prc + data << r.gujiabds_wthr_mnl_val + data << r.gujiabds_lmth_fnl_prc + data << r.data_crt_tm + data << r.gujiabdsnervetuntwprc + data << r.gujia_prc_src + data << r.gujiabdsabtjlcmbitval + data << r.gujiabdsabtjlgcaseavg + data << r.gujia_indx_lbl + data << r.gujia_prim_pty_tp + data << r.gujia_comm_tp_val + data << r.gujiabds_ltst_mdl_prc + data << r.gujia_ltst_mdl_dt + data << r.gujiabdsidxnmo_moonmo + data << r.gujiabdsa1yiminmmonmo + data << r.gujiabdsidxnyr_moonmo + data << r.gujiabdsat1yspdgrscor + data << r.gujiabdsarnd1kscavprc + allRows << data + }); + } + createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false) + } + + def exportCommCaseDetailGj(){ + def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } +// for(def city : citys){ +// def cityCode = cityCodeMap.get(city) +// def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}"; +// def allRows = [] +// pgsqlInstance.eachRow(sql,{r-> +// def data = [] +// data << r.gujia_fsid +// data << r.urbn_id +// data << r.gujia_comm_id +// data << r.gujia_case_id +// data << r.gujia_bds_crterm +// data << r.data_crt_tm +// allRows << data +//// if(allRows.size()%200000==0){ +//// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); +//// allRows.clear() +//// } +// }); +// +// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); +// } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def exportMd5(){ + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = linux_prefix + "md5.txt" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "md5.txt" + } + createFileToFtp("md5.txt",fileRows, fileName, curDate, false) + } + + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) + } + } + def fileName = linux_prefix + "/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "ok文件上传Ftp完成......" + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" 上传Ftp完成......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 7447a6712..cee28b12f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -128,9 +128,9 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> -// if(["南京"].contains(k.toString())){ + if(["漳州"].contains(k.toString())){ CITYS << k.toString(); -// } + } } // 所有城市的人口信息 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index e6cdef564..1b7d3b371 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.gj +import com.fangjia.core.BlockData +import com.fangjia.core.RegionData import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.linxsh.util.AddressUtil @@ -337,11 +339,12 @@ class FiledStrand { return null } - static def processBlock(def block){ + static def processBlock(city, region, def block){ if(block){ if(block.contains("无") || block.toString().length()>10){ return null } +// block = BlockData.formatBlock(city, region, block); return block } return null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 2806a48f1..a04bae008 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CommonDatautils.findAllCity().reverse() + def citys = ["漳州"]//CommonDatautils.findAllCity().reverse() def analysRets = [] @@ -279,7 +279,7 @@ class DistrictFieldAddBatch extends BaseBatch{ newVal = FiledStrand.processAddress(esVal, dist.city) } if(seaweedfield =="block"){ - newVal = FiledStrand.processBlock(esVal) + newVal = FiledStrand.processBlock(dist.city, dist.region, esVal) } if(seaweedfield == "build_type"){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 319e44ef4..a59b0747a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -26,8 +26,10 @@ class SeaweedDataMatchProcess { rMap.region = obj.region rMap.alias = obj.alias rMap.name = obj.name - rMap.lat = obj.lat/10000000000.0 - rMap.lng = obj.lng/10000000000.0 + if(obj.lat){ + rMap.lat = obj.lat/10000000000.0 + rMap.lng = obj.lng/10000000000.0 + } rMap.id = obj._id.toString() rMap.address = obj.address } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 91f635c3f..4961abd56 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + //广州,佛山 重庆 static main(args) { def task = new BeiKeCrawlByChrome(); task.run() @@ -447,7 +448,7 @@ public class BeiKeCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(10000) }catch(Exception e){ - Thread.sleep(60000) + Thread.sleep(10000) } return driver } From 495d959e70d8c8f169caea5eecb1250d422ced02 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 22 May 2023 17:46:20 +0800 Subject: [PATCH 24/50] =?UTF-8?q?=E6=94=B9=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pgsql/ExportAllPgData.groovy | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy index 6e9afa65b..b7a80e140 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -17,7 +17,8 @@ class ExportAllPgData { def citys = null def cityCodeMap = CcbCommonUtils.findCityCodeValue() - def database = "base" +// def database = "base" + def database = "t_app_p9pha_plda." def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def linux_prefix = "/home/goojia/ccb_data/evaluation/" def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" @@ -31,24 +32,24 @@ class ExportAllPgData { citys = CcbCommonUtils.getRunningCitys(args) def new_dists = [] - exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() +//// +//// //第三方网站权重表 +// exportWebsiteRate(); // -// //第三方网站权重表 - exportWebsiteRate(); - - exportGuidedPrice()//随人工改变// - exportCommunityGuidedPrice()//随政策// - //处理小区情况 - exportCommunityStaging()//没cityid - exportBigdataCommunity()// - - exportCommunityBuildType()// - - exportThirdPartyPrice2(new_dists)// - exportBigdataForsale(new_dists) //现在按updatetime导出 - exportBigdataDeal(new_dists) //2022-07-01 +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice()//随政策// +// //处理小区情况 +// exportCommunityStaging()//没cityid +// exportBigdataCommunity()// +// +// exportCommunityBuildType()// +// +// exportThirdPartyPrice2(new_dists)// +// exportBigdataForsale(new_dists) //现在按updatetime导出 +// exportBigdataDeal(new_dists) //2022-07-01 //计算结果相关 From 080d56715bb15d2dfa7525e1faff275db21b3f9f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 23 May 2023 09:34:52 +0800 Subject: [PATCH 25/50] update --- .../pgsql/ExportAllPgData.groovy | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy index b7a80e140..0a01037d3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -53,17 +53,17 @@ class ExportAllPgData { //计算结果相关 - exportAvmEvaluationGJNB() - exportAVMEvaluationItemGJNB() - - exportCityPrice() - exportRegionPrice() +// exportAvmEvaluationGJNB() +// exportAVMEvaluationItemGJNB() +// +// exportCityPrice() +// exportRegionPrice() exportCommunityBasePrice() // exportCommunityBasePrice1(curMonth) - exportCommunityPdescribeGj() - exportCommCaseDetailGj() +// exportCommunityPdescribeGj() +// exportCommCaseDetailGj() // exportInOk() // @@ -795,6 +795,13 @@ class ExportAllPgData { def exportCommunityBasePrice(){ + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -821,18 +828,18 @@ class ExportAllPgData { data << r.data_udt_tm data << r.gujia_src_co allRows << data - + if(allRows.size()%200000==0){ + createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, true) + createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, true) + allRows.clear() + } }); } - def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(allRows.size()>0){ + createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, true) + createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, true) } - createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false) - createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false) } From ac0ffde9c348a6a11ab2b2ccfd37c082172e77a3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 23 May 2023 11:01:43 +0800 Subject: [PATCH 26/50] update --- .../ccbevaluation/pgsql/ExportAllPgData.groovy | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy index 0a01037d3..c27bb890e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -43,7 +43,7 @@ class ExportAllPgData { // exportCommunityGuidedPrice()//随政策// // //处理小区情况 // exportCommunityStaging()//没cityid -// exportBigdataCommunity()// + exportBigdataCommunity()// // // exportCommunityBuildType()// // @@ -802,8 +802,8 @@ class ExportAllPgData { fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } - def allRows = [] for(def city : citys) { + def allRows = [] def cityCode = cityCodeMap.get(city) def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// pgsqlInstance.eachRow(sql, { r -> @@ -829,17 +829,19 @@ class ExportAllPgData { data << r.gujia_src_co allRows << data if(allRows.size()%200000==0){ - createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, true) - createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, true) + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); allRows.clear() } }); - } - if(allRows.size()>0){ - createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, true) - createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, true) + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + } } + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) } From 7587df78b74fed51ddd1a9045c07d593ee5522b3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 23 May 2023 11:05:26 +0800 Subject: [PATCH 27/50] update --- .../fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy index c27bb890e..90736c8cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -257,7 +257,7 @@ class ExportAllPgData { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from "+database+".t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from base.t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) pgsqlInstance.eachRow(sql,{r-> def data = [] data << r.urbn_id From 65c86f1e421d5402bbbd9c661940a27d845388be Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 24 May 2023 16:39:12 +0800 Subject: [PATCH 28/50] test --- .../pgsql/ExportAllPgData.groovy | 94 ++++++++++--------- 1 file changed, 52 insertions(+), 42 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy index 90736c8cc..58407d269 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -23,6 +23,8 @@ class ExportAllPgData { def linux_prefix = "/home/goojia/ccb_data/evaluation/" def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + def testCitys = ['110000','130200','140200','150200','320500','320800','330100','420900','440100','610100'] + static main(args) { def task = new ExportAllPgData() task.run(args) @@ -43,12 +45,12 @@ class ExportAllPgData { // exportCommunityGuidedPrice()//随政策// // //处理小区情况 // exportCommunityStaging()//没cityid - exportBigdataCommunity()// +// exportBigdataCommunity()// // // exportCommunityBuildType()// // -// exportThirdPartyPrice2(new_dists)// -// exportBigdataForsale(new_dists) //现在按updatetime导出 + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //现在按updatetime导出 // exportBigdataDeal(new_dists) //2022-07-01 @@ -65,10 +67,10 @@ class ExportAllPgData { // exportCommunityPdescribeGj() // exportCommCaseDetailGj() // - exportInOk() -// - SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); +// exportInOk() +//// +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } def exportEvaluationRateData() { @@ -371,8 +373,12 @@ class ExportAllPgData { def rets2 = getThirdPartyDatas(sql2) allRows.addAll(rets2) }else{ - def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' "; - allRows = getThirdPartyDatas(sql) + + if(testCitys.contains(cityCode)){ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' "; + allRows = getThirdPartyDatas(sql) + } + } if(allRows && allRows.size()>0){ index = index + 1 @@ -533,9 +539,11 @@ class ExportAllPgData { def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ - def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + - "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' "; - allRows = getHouseDatas(sql) + if(testCitys.contains(cityCode)) { + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' "; + allRows = getHouseDatas(sql) + } } println city +" "+allRows.size() if(allRows.size()>0){ @@ -805,39 +813,41 @@ class ExportAllPgData { for(def city : citys) { def allRows = [] def cityCode = cityCodeMap.get(city) - def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// - pgsqlInstance.eachRow(sql, { r -> - def data = [] - data << r.gujia_bss_prc_id - data << r.urbn_id - data << r.urbn_nm - data << r.gujia_admndiv_id - data << r.gujia_admndiv_nm - data << r.gujia_bds_id - data << r.comm_nm - data << r.gujiabds_clc_yrmo_val - data << r.gujia_bds_clc_avprc - data << r.gujia_comm_basic_prc - data << r.gujia_bds_clmtd - data << r.gujiabds_fwd_samp_rto - data << r.gujia_bds_mns_grd - data << r.gujia_bds_cfdrg - data << r.gujiabdsarndcmprccrlt - data << r.gujia_bds_cfdc_dgr - data << r.data_crt_tm - data << r.data_udt_tm - data << r.gujia_src_co - allRows << data - if(allRows.size()%200000==0){ + if(testCitys.contains(cityCode)){ + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + + if(allRows.size()>0){ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); - allRows.clear() } - }); - - if(allRows.size()>0){ - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); - FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); } } CcbFtpTestUtils.ftpUpload(null, fileName, curDate) From 177a19636df3412dda4155e219f6b99f06d14297 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 24 May 2023 17:41:53 +0800 Subject: [PATCH 29/50] =?UTF-8?q?=E5=86=99=E5=AD=97=E6=A5=BC=E5=95=86?= =?UTF-8?q?=E9=93=BA=E5=AD=97=E6=AE=B5=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/dataAna/CityInfoAnaBatch.groovy | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index cee28b12f..d008b0f48 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -69,27 +69,24 @@ class CityInfoAnaBatch extends BaseBatch { alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' ] + //out_pics : 'outPics', pics : 'pics',household_count :'householdCount',build_structure :'buildStructure', facilities : 'facilities',characteristic:'characteristic',licence:'licence',sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany', +//conditioner_time:'conditionerTime',network_way:'networkWay',fitment_level:'fitmentLevel',lighting_ventilation:'lightingVentilation', public static final def OFFICE_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', - car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure', - facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo',licence:'licence', - sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany',estate_charge:'estateCharge', - floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife', - conditioner_time:'conditionerTime',elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount', - goods_elevator_count:'goodsElevatorCount',network_way:'networkWay',fitment_level:'fitmentLevel',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea', - is_foreign:'foreign',is_register:'register',lighting_ventilation:'lightingVentilation',lobby_height:'lobbyHeight' + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate', + car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias',lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo', + estate_charge:'estateCharge',floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife', + elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount',goods_elevator_count:'goodsElevatorCount',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea', + is_foreign:'foreign',is_register:'register',lobby_height:'lobbyHeight',township:'township',divisible :'divisible',land_floor:'landFloor',underground_floor:'undergroundFloor' ] + // out_pics : 'outPics', pics : 'pics', public static final def SHOP_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', - all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany', - rent_type :'rentType',lease_term :'leaseTerm',car_site :'carSite',estate_charge :'estateCharge',divisible :'divisible',floor :'floor', - finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate', - bay_area :'bayArea', property_life :'propertyLife',unified_manage :'unifiedManage',licence :'licence',business_status :'businessStatus', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics' - + name :'name',region :'region',block :'block',address :'address', alias:'alias',addr_alias:'addrAlias',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',car_site :'carSite',estate_charge :'estateCharge',floor :'floor', + finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate',bay_area :'bayArea', property_life :'propertyLife',business_status :'businessStatus',parking_fee:'parkingFee', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2',township:'township',loop_line_info :'loopLineInfo',land_floor:'landFloor',underground_floor:'undergroundFloor',real_use_percent:'realUsePercent' ] public static CITY_GRADE_MAP = [1000: "一线" , 1500 : "新一线" , 2000 : "二线" , 3000:"三线" ,4000 : "四线" ,5000 : "五线"] @@ -128,9 +125,9 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> - if(["漳州"].contains(k.toString())){ +// if(["南京"].contains(k.toString())){ CITYS << k.toString(); - } +// } } // 所有城市的人口信息 From a794ef50a3359f81bf907d8719e01232b6159ccb Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 25 May 2023 11:13:20 +0800 Subject: [PATCH 30/50] =?UTF-8?q?=E5=95=86=E9=93=BA=E5=86=99=E5=AD=97?= =?UTF-8?q?=E6=A5=BC=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkoff/controllers/DataController.java | 18 + .../controllers/District2Controller.java | 6 +- .../src/main/webapp/district2/add.ftl | 229 ++++--- .../src/main/webapp/district2/edit.ftl | 588 ++++++++++-------- 4 files changed, 467 insertions(+), 374 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java index 26dfcdb1f..3d55ac818 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DataController.java @@ -1015,6 +1015,24 @@ public class DataController { put("mainType","主力户型"); put("mappingAlias","映射别名"); put("districtAliasTotalCount", "别名总量"); + + put("undergroundCarSite", "地下车位数"); + put("landCarSite", "地上车位数"); + put("parkingFee", "车位月租金"); + put("sight", "开间面积"); + put("lobbyHeight", "大堂层高"); + put("standardHeight", "标准层高"); + put("clearHeight", "净高"); + put("standardArea", "标准层面积"); + put("foreign", "是否涉外"); + put("register", "是否可注册"); + put("divisible", "是否可分割"); + put("elevatorsCount","电梯数量"); + put("personElevatorsCount","客梯数"); + put("goodsElevatorCount","货梯数"); + put("realUsePercent","得房率"); + put("landFloor","地上层数"); + put("undergroundFloor","地下层数"); } };// return map; } diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java index 52a578dbd..ff1ff0425 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java @@ -832,10 +832,10 @@ public class District2Controller { }else if(district.getCategory().equals("shop")){ data.put("propertyStatus", district.getPropertyStatus()); data.put("shopEstateType2", district.getShopEstateType2()); + data.put("landFloor", district.getLandFloor()); + data.put("undergroundFloor", district.getUndergroundFloor()); + data.put("realUsePercent", district.getRealUsePercent()); } - - - districtService.updateById(data, district.getId()); msg="保存成功!"; } diff --git a/projects/fangjia-bkoff/src/main/webapp/district2/add.ftl b/projects/fangjia-bkoff/src/main/webapp/district2/add.ftl index bef1afb70..1d2094b22 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district2/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district2/add.ftl @@ -25,7 +25,7 @@ - + @@ -123,46 +123,52 @@ - - - - - + +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - - - - - - - - - - + @@ -203,6 +209,19 @@ 灞� + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> + +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - + +<#-- --> +<#-- --> +<#-- --> +<#-- -->
绫诲瀷*绫诲瀷* 鍟嗛摵
浜岀骇鍒嗙被 - 鍟嗗姟鍖哄晢閾�   - 浜ら�氳鏂藉晢閾�   - 甯傚満绫诲晢閾�   - 绀惧尯鍐呴摵   - 浣忓畢搴曞晢   - 鐧捐揣璐墿涓績鍟嗛摵   - 鍟嗕笟缁煎悎浣�   - -
寤虹瓚鐘舵��涔¢晣琛楅亾 - +
浜岀骇鍒嗙被--> +<#-- 鍟嗗姟鍖哄晢閾�  --> +<#-- 浜ら�氳鏂藉晢閾�  --> +<#-- 甯傚満绫诲晢閾�  --> +<#-- 绀惧尯鍐呴摵  --> +<#-- 浣忓畢搴曞晢  --> +<#-- 鐧捐揣璐墿涓績鍟嗛摵  --> +<#-- 鍟嗕笟缁煎悎浣�  --> +<#-- --> +<#--
寤虹瓚鐘舵��--> +<#-- --> +<#--
绉熷敭绫诲瀷--> +<#-- --> +<#--
绉熻祦骞撮檺--> +<#-- --> +<#--
浠樻鏂瑰紡--> +<#-- --> +<#--
绉熷敭绫诲瀷 - -
绉熻祦骞撮檺 - -
浠樻鏂瑰紡 - -
鍋滆溅浣嶆暟鍋滆溅浣嶆暟 涓�
鍦颁笂灞傛暟 + 灞� +
鍦颁笅灞傛暟 + 灞� +
鍗犲湴闈㈢Н @@ -251,71 +270,71 @@
鏄惁鍙垎鍓� - -
鏄惁缁熶竴绠$悊 - -
鍦伴搧娌跨嚎 - -
鍦伴搧绔欏悕 - -
瑁呬慨鎯呭喌 - -
鐗╀笟姒傚喌 - -
浜ら�氭儏鍐� - -
鍛ㄨ竟鎯呭喌 - -
閰嶅璁炬柦 - -
鎻忚堪 - -
鏄惁鍙垎鍓�--> +<#-- --> +<#--
鏄惁缁熶竴绠$悊--> +<#-- --> +<#--
鍦伴搧娌跨嚎--> +<#-- --> +<#--
鍦伴搧绔欏悕--> +<#-- --> +<#--
瑁呬慨鎯呭喌--> +<#-- --> +<#--
鐗╀笟姒傚喌--> +<#-- --> +<#--
浜ら�氭儏鍐�--> +<#-- --> +<#--
鍛ㄨ竟鎯呭喌--> +<#-- --> +<#--
閰嶅璁炬柦--> +<#-- --> +<#--
鎻忚堪--> +<#-- --> +<#--
缁忚惀涓氭�� @@ -323,11 +342,17 @@
椤圭洰鐗硅壊寰楁埧鐜� - +
椤圭洰鐗硅壊--> +<#-- --> +<#--
diff --git a/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl index 4caf1d803..c0035175a 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl @@ -25,7 +25,7 @@ - - - - + + - + +<#-- --> +<#-- --> +<#-- --> +<#-- -->
+ <#if markMap?? && markMap["cat"]?? && markMap["cat"] == 1> <#else> @@ -232,55 +232,67 @@
<#if markMap?? && markMap["shop_estate_type2"]?? && markMap["shop_estate_type2"] == 1> - - <#else> - - 浜岀骇鍒嗙被 - <#assign x1 = 0> - <#assign x2 = 0> - <#assign x3 = 0> - <#assign x4 = 0> - <#assign x5 = 0> - <#assign x6 = 0> - <#assign x7 = 0> - <#if district?? && district.shopEstateType2??> - <#list district.shopEstateType2 as l> - <#if l == "鍟嗗姟鍖哄晢閾�"> - <#assign x1 = 1> - - <#if l == "浜ら�氳鏂藉晢閾�"> - <#assign x2 = 1> - - <#if l == "甯傚満绫诲晢閾�"> - <#assign x3 = 1> - - <#if l == "绀惧尯鍐呴摵"> - <#assign x4 = 1> - - <#if l == "浣忓畢搴曞晢"> - <#assign x5 = 1> - - <#if l == "鐧捐揣璐墿涓績鍟嗛摵"> - <#assign x6 = 1> - - <#if l == "鍟嗕笟缁煎悎浣�"> - <#assign x7 = 1> - - - - checked="checked"/>鍟嗗姟鍖哄晢閾�   - checked="checked"/>浜ら�氳鏂藉晢閾�   - checked="checked"/>甯傚満绫诲晢閾�   - checked="checked"/>绀惧尯鍐呴摵   - checked="checked"/>浣忓畢搴曞晢   - checked="checked"/>鐧捐揣璐墿涓績鍟嗛摵   - checked="checked"/>鍟嗕笟缁煎悎浣�   - +
+ <#if markMap?? && markMap["township"]?? && markMap["township"]?is_number && markMap["township"] == 1> + + <#else> + + + 涔¢晣琛楅亾 +
<#if markMap?? && markMap["shop_estate_type2"]?? && markMap["shop_estate_type2"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- 浜岀骇鍒嗙被--> +<#-- <#assign x1 = 0>--> +<#-- <#assign x2 = 0>--> +<#-- <#assign x3 = 0>--> +<#-- <#assign x4 = 0>--> +<#-- <#assign x5 = 0>--> +<#-- <#assign x6 = 0>--> +<#-- <#assign x7 = 0>--> +<#-- <#if district?? && district.shopEstateType2??>--> +<#-- <#list district.shopEstateType2 as l>--> +<#-- <#if l == "鍟嗗姟鍖哄晢閾�">--> +<#-- <#assign x1 = 1>--> +<#-- --> +<#-- <#if l == "浜ら�氳鏂藉晢閾�">--> +<#-- <#assign x2 = 1>--> +<#-- --> +<#-- <#if l == "甯傚満绫诲晢閾�">--> +<#-- <#assign x3 = 1>--> +<#-- --> +<#-- <#if l == "绀惧尯鍐呴摵">--> +<#-- <#assign x4 = 1>--> +<#-- --> +<#-- <#if l == "浣忓畢搴曞晢">--> +<#-- <#assign x5 = 1>--> +<#-- --> +<#-- <#if l == "鐧捐揣璐墿涓績鍟嗛摵">--> +<#-- <#assign x6 = 1>--> +<#-- --> +<#-- <#if l == "鍟嗕笟缁煎悎浣�">--> +<#-- <#assign x7 = 1>--> +<#-- --> +<#-- --> +<#-- --> +<#-- checked="checked"/>鍟嗗姟鍖哄晢閾�  --> +<#-- checked="checked"/>浜ら�氳鏂藉晢閾�  --> +<#-- checked="checked"/>甯傚満绫诲晢閾�  --> +<#-- checked="checked"/>绀惧尯鍐呴摵  --> +<#-- checked="checked"/>浣忓畢搴曞晢  --> +<#-- checked="checked"/>鐧捐揣璐墿涓績鍟嗛摵  --> +<#-- checked="checked"/>鍟嗕笟缁煎悎浣�  --> +<#-- --> +<#--
@@ -292,56 +304,56 @@
+<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> - - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - - - - +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> + <#if markMap?? && markMap["business_status"]?? && markMap["business_status"] == 1> + + <#else> + + + 缁忚惀涓氭�� + + + + +
--> +<#-- <#if markMap?? && markMap["property_status"]?? && markMap["property_status"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 寤虹瓚鐘舵��--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["rent_type"]?? && markMap["rent_type"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 绉熷敭绫诲瀷--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["lease_term"]?? && markMap["lease_term"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 绉熻祦骞撮檺--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["payment_pattern"]?? && markMap["payment_pattern"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 浠樻鏂瑰紡--> +<#-- --> +<#--
- <#if markMap?? && markMap["property_status"]?? && markMap["property_status"] == 1> - - <#else> - - - 寤虹瓚鐘舵�� - -
- <#if markMap?? && markMap["rent_type"]?? && markMap["rent_type"] == 1> - - <#else> - - - 绉熷敭绫诲瀷 - -
- <#if markMap?? && markMap["lease_term"]?? && markMap["lease_term"] == 1> - - <#else> - - - 绉熻祦骞撮檺 - -
- <#if markMap?? && markMap["payment_pattern"]?? && markMap["payment_pattern"] == 1> - - <#else> - - - 浠樻鏂瑰紡 - -
+ <#if markMap?? && markMap["car_site_count"]?? && markMap["car_site_count"] == 1> <#else> @@ -421,6 +433,30 @@
+ <#if markMap?? && markMap["land_floor"]?? && markMap["land_floor"] == 1> + + <#else> + + + 鍦颁笂妤煎眰 + 灞� +
+ <#if markMap?? && markMap["underground_floor"]?? && markMap["underground_floor"] == 1> + + <#else> + + + 鍦颁笅妤煎眰 + 灞� +
<#if markMap?? && markMap["all_area"]?? && markMap["all_area"] == 1> @@ -541,63 +577,88 @@
- <#if markMap?? && markMap["sideset"]?? && markMap["sideset"] == 1> - - <#else> - - - 鏄惁鍙垎鍓� - -
- <#if markMap?? && markMap["unified_manage"]?? && markMap["unified_manage"] == 1> - - <#else> - - - 鏄惁缁熶竴绠$悊 - - -
- <#if markMap?? && markMap["metro_info"]?? && markMap["metro_info"] == 1> - - <#else> - - - 鍦伴搧娌跨嚎 - -
--> +<#-- <#if markMap?? && markMap["sideset"]?? && markMap["sideset"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鏄惁鍙垎鍓�--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["unified_manage"]?? && markMap["unified_manage"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鏄惁缁熶竴绠$悊--> +<#-- --> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["metro_info"]?? && markMap["metro_info"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鍦伴搧娌跨嚎--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["metro_names"]?? && markMap["metro_names"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鍦伴搧绔欏悕--> +<#-- --> +<#--
- <#if markMap?? && markMap["metro_names"]?? && markMap["metro_names"] == 1> - - <#else> - - - 鍦伴搧绔欏悕 - + +
+ <#if markMap?? && markMap["real_use_percent"]?? && markMap["real_use_percent"] == 1> + + <#else> + + + 寰楁埧鐜� +
@@ -609,125 +670,114 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- <#if markMap?? && markMap["fitment"]?? && markMap["fitment"] == 1> - - <#else> - - - 瑁呬慨鎯呭喌 - -
- <#if markMap?? && markMap["build_structure"]?? && markMap["build_structure"] == 1> - - <#else> - - - 寤虹瓚缁撴瀯 - -
- <#if markMap?? && markMap["property_info"]?? && markMap["property_info"] == 1> - - <#else> - - - 鐗╀笟姒傚喌 - -
- <#if markMap?? && markMap["traffic"]?? && markMap["traffic"] == 1> - - <#else> - - - 浜ら�氭鍐� - -
- <#if markMap?? && markMap["surrounding"]?? && markMap["surrounding"] == 1> - - <#else> - - - 鍛ㄨ竟鎯呭喌 - -
- <#if markMap?? && markMap["facilities"]?? && markMap["facilities"] == 1> - - <#else> - - - 閰嶅璁炬柦 - -
- <#if markMap?? && markMap["description"]?? && markMap["description"] == 1> - - <#else> - - - 鎻忚堪 - -
- <#if markMap?? && markMap["business_status"]?? && markMap["business_status"] == 1> - - <#else> - - - 缁忚惀涓氭�� - -
- <#if markMap?? && markMap["characteristic"]?? && markMap["characteristic"] == 1> - - <#else> - - - 椤圭洰鐗硅壊 - -
+<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> + +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["fitment"]?? && markMap["fitment"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 瑁呬慨鎯呭喌--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["build_structure"]?? && markMap["build_structure"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 寤虹瓚缁撴瀯--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["property_info"]?? && markMap["property_info"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鐗╀笟姒傚喌--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["traffic"]?? && markMap["traffic"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 浜ら�氭鍐�--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["surrounding"]?? && markMap["surrounding"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鍛ㄨ竟鎯呭喌--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["facilities"]?? && markMap["facilities"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 閰嶅璁炬柦--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["description"]?? && markMap["description"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鎻忚堪--> +<#-- --> +<#--
--> +<#-- <#if markMap?? && markMap["characteristic"]?? && markMap["characteristic"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 椤圭洰鐗硅壊--> +<#-- --> +<#--
--> - - - - -
- -
+<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#--
--> +<#-- --> +<#--
--> From e216f45965cc1c3df88bd6ea61cf52159bc81c70 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 25 May 2023 10:28:08 +0800 Subject: [PATCH 31/50] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=B2=BB=E7=90=86=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AddressGovernanceController.java | 259 ++++++++++++++++++ .../dto/addressGovernance/DistrictDto.java | 50 ++++ .../DistrictIdentifyDto.java | 101 +++++++ .../addressGovernance/MatchedBuildingDto.java | 128 +++++++++ .../StructureAndCompleteDto.java | 203 ++++++++++++++ .../dto/addressGovernance/StructureDto.java | 191 +++++++++++++ .../com/fangjia/open/model/AddressModel.java | 21 +- .../com/fangjia/open/util/CommonDtoUtils.java | 41 +++ 8 files changed, 993 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictDto.java create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictIdentifyDto.java create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureAndCompleteDto.java create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureDto.java create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java new file mode 100644 index 000000000..f7b63a250 --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java @@ -0,0 +1,259 @@ +package com.fangjia.open.controllers; + +import com.fangjia.core.util.HttpRequest; +import com.fangjia.open.dto.addressGovernance.*; +import com.fangjia.open.model.AddesssMatchModel; +import com.fangjia.open.model.AddressModel; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; +import com.fangjia.open.util.CommonDtoUtils; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * 地址治理 + * + * @author shenjian + * @since 2023/5/25 + */ +//@Authority(role = Role.USER) +public class AddressGovernanceController { + + /** + * 地址结构化 + * + * @param model + * @return + */ + public Object structureAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", model.getCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure/v2", params, "UTF-8"); + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(CommonDtoUtils.transformV(StructureDto.class, json.get("result"))); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + } catch(Exception e){ + e.printStackTrace(); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 区划识别接口 + * + * @param model + * @return + */ + public Object fiveDivisionStructureAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", model.getCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8"); + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(CommonDtoUtils.transformV(DistrictIdentifyDto.class, json.get("result"))); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + } catch(Exception e){ + e.printStackTrace(); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + + + return mav; + } + + /** + * 地址补全 + */ + public Object structureAndCompleteAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", model.getCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8"); + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(CommonDtoUtils.transformV(StructureAndCompleteDto.class, json.get("result"))); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + } catch(Exception e){ + e.printStackTrace(); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 地址验证 + */ + public Object addrVerifyAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", model.getCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/addrVerify", params, "UTF-8"); + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + } catch(Exception e){ + e.printStackTrace(); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 地址匹配 + */ + public Object matchAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", model.getCity()); + params.put("address", model.getAddress()); + params.put("category", model.getCategory()); + params.put("district", model.getSuggestDistrict()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/match", params, "UTF-8"); + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(CommonDtoUtils.transformV(MatchedBuildingDto.class, json.get("result"))); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + } catch(Exception e){ + e.printStackTrace(); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + + + return mav; + } + + /** + * 地址联想 + */ + public Object addressAssociationAction(AddressModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getCity())) { + mav.setModel(BasicModel.buildBadRequestModel("城市不能为空")); + return mav; + } + if (StringUtils.isBlank(model.getKeyword())) { + mav.setModel(BasicModel.buildBadRequestModel("关键字不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", model.getCity()); + params.put("region", model.getRegion()); + params.put("keywords", model.getKeyword()); + params.put("page", "0"); + params.put("size", String.valueOf(model.getMax() == 0 ? 20 : (model.getMax() > 100 ? 100 : model.getMax()) )); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/addressAssociation2", params, "UTF-8"); + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(CommonDtoUtils.transformListV(DistrictDto.class, (JSONObject.fromObject(json.get("result"))).getJSONArray("content"))); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + } catch(Exception e){ + e.printStackTrace(); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictDto.java new file mode 100644 index 000000000..cc71dcf21 --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictDto.java @@ -0,0 +1,50 @@ +package com.fangjia.open.dto.addressGovernance; + +public class DistrictDto { + + private String province; + private String city; + private String region; + private String name; + private String address; + + public String getProvince() { + return province; + } + + public void setProvince(String province) { + this.province = province; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } +} \ No newline at end of file diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictIdentifyDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictIdentifyDto.java new file mode 100644 index 000000000..03301c218 --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/DistrictIdentifyDto.java @@ -0,0 +1,101 @@ +package com.fangjia.open.dto.addressGovernance; + +/** + * 区划识别DTO + * + * @author shenjian + * @since 2023/5/25 + */ +public class DistrictIdentifyDto { + + private String province; + private String provinceCode; + private String city; + private String cityCode; + private String district; + private String districtCode; + private String township; + private String townshipCode; + private String community; + private String communityCode; + + public String getProvince() { + return province; + } + + public void setProvince(String province) { + this.province = province; + } + + public String getProvinceCode() { + return provinceCode; + } + + public void setProvinceCode(String provinceCode) { + this.provinceCode = provinceCode; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getCityCode() { + return cityCode; + } + + public void setCityCode(String cityCode) { + this.cityCode = cityCode; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getDistrictCode() { + return districtCode; + } + + public void setDistrictCode(String districtCode) { + this.districtCode = districtCode; + } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } + + public String getTownshipCode() { + return townshipCode; + } + + public void setTownshipCode(String townshipCode) { + this.townshipCode = townshipCode; + } + + public String getCommunity() { + return community; + } + + public void setCommunity(String community) { + this.community = community; + } + + public String getCommunityCode() { + return communityCode; + } + + public void setCommunityCode(String communityCode) { + this.communityCode = communityCode; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java new file mode 100644 index 000000000..6568ae6bd --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java @@ -0,0 +1,128 @@ +package com.fangjia.open.dto.addressGovernance; + +/** + * 鍦板潃鍖归厤DTO + */ +public class MatchedBuildingDto { + + /** + * ID + */ + private String id; + /** + * 缃俊搴� + */ + private Double credit; + /** + * 鍩庡競 + */ + private String city; + /** + * 鍖哄煙 + */ + private String district; + /** + * 涔¢晣琛楅亾 + */ + private String township; + /** + * 寤虹瓚鐗╁悕绉� + */ + private String name; + /** + * 寤虹瓚鐗╁湴鍧� + */ + private String address; + /** + * 缁忓害(楂樺痉) + */ + private Double lng; + /** + * 绾害(楂樺痉) + */ + private Double lat; + /** + * 鐗╀笟绫诲瀷 + */ + private String propertyType; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Double getCredit() { + return credit; + } + + public void setCredit(Double credit) { + this.credit = credit; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public Double getLng() { + return lng; + } + + public void setLng(Double lng) { + this.lng = lng; + } + + public Double getLat() { + return lat; + } + + public void setLat(Double lat) { + this.lat = lat; + } + + public String getPropertyType() { + return propertyType; + } + + public void setPropertyType(String propertyType) { + this.propertyType = propertyType; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureAndCompleteDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureAndCompleteDto.java new file mode 100644 index 000000000..6407c8ce9 --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureAndCompleteDto.java @@ -0,0 +1,203 @@ +package com.fangjia.open.dto.addressGovernance; + +/** + * 地址补全DTO + * + * @author shenjian + * @since 2023/5/25 + */ +public class StructureAndCompleteDto { + + /** + * 省份 + */ + private String province; + /** + * 城市 + */ + private String city; + /** + * 区域 + */ + private String district; + /** + * 开发区 + */ + private String dev; + /** + * 乡镇街道 + */ + private String township; + /** + * 道路 + */ + private String street; + /** + * 路号 + */ + private String streetNumber; + /** + * 居委村委 + */ + private String community; + /** + * 村组 + */ + private String group; + /** + * 名称 + */ + private String name; + /** + * 楼栋 + */ + private String buildingNumber; + /** + * 单元 + */ + private String unitNumber; + /** + * 楼层 + */ + private String floor; + /** + * 室号 + */ + private String roomNumber; + /** + * 格式化地址 + */ + private String formatAddress; + /** + * 地址验证 + */ + private String addrVerify; + + public String getProvince() { + return province; + } + + public void setProvince(String province) { + this.province = province; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getDev() { + return dev; + } + + public void setDev(String dev) { + this.dev = dev; + } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } + + public String getStreet() { + return street; + } + + public void setStreet(String street) { + this.street = street; + } + + public String getStreetNumber() { + return streetNumber; + } + + public void setStreetNumber(String streetNumber) { + this.streetNumber = streetNumber; + } + + public String getCommunity() { + return community; + } + + public void setCommunity(String community) { + this.community = community; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getBuildingNumber() { + return buildingNumber; + } + + public void setBuildingNumber(String buildingNumber) { + this.buildingNumber = buildingNumber; + } + + public String getUnitNumber() { + return unitNumber; + } + + public void setUnitNumber(String unitNumber) { + this.unitNumber = unitNumber; + } + + public String getFloor() { + return floor; + } + + public void setFloor(String floor) { + this.floor = floor; + } + + public String getRoomNumber() { + return roomNumber; + } + + public void setRoomNumber(String roomNumber) { + this.roomNumber = roomNumber; + } + + public String getFormatAddress() { + return formatAddress; + } + + public void setFormatAddress(String formatAddress) { + this.formatAddress = formatAddress; + } + + public String getAddrVerify() { + return addrVerify; + } + + public void setAddrVerify(String addrVerify) { + this.addrVerify = addrVerify; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureDto.java new file mode 100644 index 000000000..f2ca60475 --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/StructureDto.java @@ -0,0 +1,191 @@ +package com.fangjia.open.dto.addressGovernance; + +/** + * 地址结构化,请不要增加其他字段,与文档及地数输出一致 + * + * @author shenjian + * @since 2023/5/25 + */ +public class StructureDto { + + /** + * 省份 + */ + private String province; + /** + * 城市 + */ + private String city; + /** + * 区域 + */ + private String district; + /** + * 开发区 + */ + private String dev; + /** + * 乡镇街道 + */ + private String township; + /** + * 道路 + */ + private String street; + /** + * 路号 + */ + private String streetNumber; + /** + * 居委村委 + */ + private String community; + /** + * 村组 + */ + private String group; + /** + * 名称 + */ + private String name; + /** + * 楼栋 + */ + private String buildingNumber; + /** + * 单元 + */ + private String unitNumber; + /** + * 楼层 + */ + private String floor; + /** + * 室号 + */ + private String roomNumber; + /** + * 地址验证 + */ + private String addrVerify; + + public String getProvince() { + return province; + } + + public void setProvince(String province) { + this.province = province; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getDev() { + return dev; + } + + public void setDev(String dev) { + this.dev = dev; + } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } + + public String getStreet() { + return street; + } + + public void setStreet(String street) { + this.street = street; + } + + public String getStreetNumber() { + return streetNumber; + } + + public void setStreetNumber(String streetNumber) { + this.streetNumber = streetNumber; + } + + public String getCommunity() { + return community; + } + + public void setCommunity(String community) { + this.community = community; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getBuildingNumber() { + return buildingNumber; + } + + public void setBuildingNumber(String buildingNumber) { + this.buildingNumber = buildingNumber; + } + + public String getUnitNumber() { + return unitNumber; + } + + public void setUnitNumber(String unitNumber) { + this.unitNumber = unitNumber; + } + + public String getFloor() { + return floor; + } + + public void setFloor(String floor) { + this.floor = floor; + } + + public String getRoomNumber() { + return roomNumber; + } + + public void setRoomNumber(String roomNumber) { + this.roomNumber = roomNumber; + } + + public String getAddrVerify() { + return addrVerify; + } + + public void setAddrVerify(String addrVerify) { + this.addrVerify = addrVerify; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java index 8c41c06ab..529d42d07 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java @@ -23,6 +23,7 @@ public class AddressModel extends FangjiaApiModel { private String preciseLevel; private String province; private String district; + private String region; private String chooseId; @@ -36,6 +37,9 @@ public class AddressModel extends FangjiaApiModel { private Integer pageNo; // 是否用于检查 private String dataLevel; + + private String keyword; + public boolean isContainsLocation() { return containsLocation; } @@ -228,5 +232,20 @@ public class AddressModel extends FangjiaApiModel { public void setDataLevel(String dataLevel) { this.dataLevel = dataLevel; } - + + public String getKeyword() { + return keyword; + } + + public void setKeyword(String keyword) { + this.keyword = keyword; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java new file mode 100644 index 000000000..f324bc8f6 --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java @@ -0,0 +1,41 @@ +package com.fangjia.open.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +import java.util.List; + +/** + * 閫氱敤瀹炰綋涓嶥TO浜掔浉杞崲宸ュ叿绫� + * + * @author shenjian + * @date 2021/06/16 + */ +public class CommonDtoUtils { + public static T transformV(Class targetClass, Object source) { + if (source == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(source); + return JSONObject.parseObject(jsonSource, targetClass); + } catch (Exception var3) { + return null; + } + } + } + + public static List transformListV(Class targetClass, List listSource) { + if (listSource == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(listSource); + return JSONArray.parseArray(jsonSource, targetClass); + } catch (Exception var3) { + return null; + } + } + } +} \ No newline at end of file From 450d09217c10ac3429af229c8e4fd44959a0f18f Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 25 May 2023 14:35:25 +0800 Subject: [PATCH 32/50] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=B2=BB=E7=90=86=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/AddressGovernanceController.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java index f7b63a250..98a343576 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java @@ -2,6 +2,8 @@ package com.fangjia.open.controllers; import com.fangjia.core.util.HttpRequest; import com.fangjia.open.dto.addressGovernance.*; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; import com.fangjia.open.model.AddesssMatchModel; import com.fangjia.open.model.AddressModel; import com.fangjia.open.model.BasicModel; @@ -19,7 +21,7 @@ import java.util.Map; * @author shenjian * @since 2023/5/25 */ -//@Authority(role = Role.USER) +@Authority(role = Role.USER) public class AddressGovernanceController { /** From 4a5e3020070f06d07c3f8a817bc954d67252dca2 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 25 May 2023 14:42:50 +0800 Subject: [PATCH 33/50] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BC=96=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../addressGovernance/MatchedBuildingDto.java | 20 +++++++++---------- .../com/fangjia/open/util/CommonDtoUtils.java | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java index 6568ae6bd..e666a9100 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/addressGovernance/MatchedBuildingDto.java @@ -1,7 +1,7 @@ package com.fangjia.open.dto.addressGovernance; /** - * 鍦板潃鍖归厤DTO + * 地址匹配DTO */ public class MatchedBuildingDto { @@ -10,39 +10,39 @@ public class MatchedBuildingDto { */ private String id; /** - * 缃俊搴� + * 置信度 */ private Double credit; /** - * 鍩庡競 + * 城市 */ private String city; /** - * 鍖哄煙 + * 区域 */ private String district; /** - * 涔¢晣琛楅亾 + * 乡镇街道 */ private String township; /** - * 寤虹瓚鐗╁悕绉� + * 建筑物名称 */ private String name; /** - * 寤虹瓚鐗╁湴鍧� + * 建筑物地址 */ private String address; /** - * 缁忓害(楂樺痉) + * 经度(高德) */ private Double lng; /** - * 绾害(楂樺痉) + * 纬度(高德) */ private Double lat; /** - * 鐗╀笟绫诲瀷 + * 物业类型 */ private String propertyType; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java index f324bc8f6..bebd942e0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/CommonDtoUtils.java @@ -7,7 +7,7 @@ import com.alibaba.fastjson.JSONObject; import java.util.List; /** - * 閫氱敤瀹炰綋涓嶥TO浜掔浉杞崲宸ュ叿绫� + * 通用实体与DTO互相转换工具类 * * @author shenjian * @date 2021/06/16 From ffe2c8a8a3e178ff7c8abfc455f615265c8601b8 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 26 May 2023 11:35:45 +0800 Subject: [PATCH 34/50] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...sGovernanceController.java => AddrSvcController.java} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename projects/fangjia-open/src/main/java/com/fangjia/open/controllers/{AddressGovernanceController.java => AddrSvcController.java} (97%) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java similarity index 97% rename from projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java rename to projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index 98a343576..d17fb74f3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressGovernanceController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -11,6 +11,7 @@ import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.open.util.CommonDtoUtils; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Controller; import java.util.HashMap; import java.util.Map; @@ -22,7 +23,7 @@ import java.util.Map; * @since 2023/5/25 */ @Authority(role = Role.USER) -public class AddressGovernanceController { +public class AddrSvcController { /** * 地址结构化 @@ -69,7 +70,7 @@ public class AddressGovernanceController { * @param model * @return */ - public Object fiveDivisionStructureAction(AddesssMatchModel model) { + public Object divisionAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress())) { @@ -107,7 +108,7 @@ public class AddressGovernanceController { /** * 地址补全 */ - public Object structureAndCompleteAction(AddesssMatchModel model) { + public Object completeAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress())) { @@ -219,7 +220,7 @@ public class AddressGovernanceController { /** * 地址联想 */ - public Object addressAssociationAction(AddressModel model) { + public Object suggestionAction(AddressModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getCity())) { From beeec6607bfac9bfae0b1662b31f1252de558eb4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 26 May 2023 18:37:10 +0800 Subject: [PATCH 35/50] update --- .../ccbevaluation/CcbCommonUtils.groovy | 13 +++++++--- .../script/commData/CommonDatautils.groovy | 26 +++++++++++++++++++ .../script/jiangnanblank/CZDataBatch.groovy | 1 + .../script/jiangnanblank/FzDataUtils.groovy | 18 +++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index d0a216521..465eac28c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -218,6 +218,7 @@ class CcbCommonUtils { q.put("avg_price", new BasicDBObject('$gt', 0)) q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) q.put("match_id", dist._id.toString()) + q.put("type", new BasicDBObject('$ne','车位')) def dbcursor = DEAL_PRICE_COLL.find(q) def total = dbcursor.count() def rets = new HashSet() @@ -445,12 +446,17 @@ class CcbCommonUtils { def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000) println "[${city}] 原始:"+dbsour.count() def rets = [] + + def ids = [] for(def d : dbsour){ if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ if(d.address){ d.address = d.address.replaceAll("\r|\n|\\s", ""); } - rets << d + if(!ids.contains(d._id.toString())){ + ids << d._id.toString() + rets << d + } } } println "[${city}] 小区量:"+rets.size() @@ -606,11 +612,12 @@ class CcbCommonUtils { - public static def findTradesByMonth(city, startDate, endDate){ + public static def findTradesByMonth(city, startDate){ def q = new BasicDBObject() q.put("city", city) - q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + q.put("trade_date", new BasicDBObject('$gte', startDate)) q.put("match_id", new BasicDBObject('$exists', false)) +// q.put("district_name","国贸天琴湾") def dbcursor = DEAL_PRICE_COLL.find(q) def total = dbcursor.count() def rets = new HashSet() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index b7bf2feb6..543973cfa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -71,6 +71,7 @@ class CommonDatautils { public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl) public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl) public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl) + public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo); public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -2989,6 +2990,19 @@ class CommonDatautils { } return names; } + + + public static def getCityByRegion(region){ + DBObject q = new BasicDBObject(); + q.put("category", "region"); + q.put("name", region); + def cursor = REGION_BLOCK_COLL.find(q); + def citys = []; + for(def obj : cursor){ + citys << obj.city; + } + return citys; + } static def findBusLineCount(def city, def maxLat, def minLat, def maxLng, def minLng){ DBObject q = new BasicDBObject() @@ -3464,5 +3478,17 @@ class CommonDatautils { } return null } + + static def findRegionPriceRate(city, region){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + def cur = REGION_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1) + def obj = null + while(cur.hasNext()) { + obj = cur.next() + } + return obj + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index d55d98dc4..ced116b61 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -59,6 +59,7 @@ class CZDataBatch extends BaseBatch{ def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); districtRun(districts, statMap, commids); + FzDataUtils.removeNoUpdate(city, districts) def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = mergeShop(shops); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index 8a9325c9b..bf6e2c177 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -101,6 +101,23 @@ class FzDataUtils { saveDistrict(map); } + static def removeNoUpdate(city, districts){ + def ids = [] + for(def d : districts){ + ids << d._id.toString() + } + DBObject q = new BasicDBObject(); + q.put("city", city) + q.put("cat", "district") + def dbcursor = DIST_COLL.find(q) + for(def cur : dbcursor){ + if(!ids.contains(cur.num_id)){ + def del_data = new BasicDBObject() + del_data.put("num_id", cur.num_id) + DIST_COLL.remove(del_data) + } + } + } static def saveDistrict(def m){ DBObject q = new BasicDBObject(); @@ -111,6 +128,7 @@ class FzDataUtils { m.each {k,v-> udata.put(k, v); } + udata.put("m_date", new Date()) if(dbo){ udata.removeField("num_id") DIST_COLL.update(q, new BasicDBObject('$set', udata), false, true); From 532503a7e345e0a266f2b8ec19cd63756c2c9feb Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 29 May 2023 11:09:04 +0800 Subject: [PATCH 36/50] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=B5=99=E5=95=86?= =?UTF-8?q?=E9=A3=8E=E9=99=A9=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 82 +++++++++++++++---- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 3b9563cbe..ad2c3adbd 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -87,7 +87,7 @@ import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -@Authority(role = Role.USER) +//@Authority(role = Role.USER) public class AddressController { private AddressService addressService; @@ -985,6 +985,14 @@ public class AddressController { return term; } + private JSONObject addMatchPhrase(String name, String value){ + JSONObject term = new JSONObject(); + JSONObject match = new JSONObject(); + match.put(name, value); + term.put("match_phrase", match); + return term; + } + /*** * 获取自然村 * @param city @@ -3279,7 +3287,7 @@ public class AddressController { return mav; } - @Authority(role = Role.USER) + //@Authority(role = Role.USER) public Object riskAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); // if (StringUtils.isEmpty(model.getCity())) { @@ -3322,20 +3330,26 @@ public class AddressController { searchKey.append(searchAddress.getString("streetNumber")); } if (StringUtils.isNotBlank(searchKey.toString()) && searchAddress.has("city") && StringUtils.isNotBlank(searchAddress.getString("city"))) { - JSONObject jsonObject = addressMatchProxyService.searchByAmap(searchAddress.getString("city"), model.getAddress()); - JSONArray poisData = jsonObject.getJSONArray("pois"); - for (int i = 0; i < poisData.size(); i++) { - String address = poisData.getJSONObject(i).getString("address"); - if (address.contains(searchKey.toString())) { - resultMap.put("riskLevel", "A"); - resultModel.setResult(resultMap); - mav.setModel(resultModel); - return mav; - } else { - continue; - } +// JSONObject jsonObject = addressMatchProxyService.searchByAmap(searchAddress.getString("city"), model.getAddress()); +// JSONArray poisData = jsonObject.getJSONArray("pois"); +// for (int i = 0; i < poisData.size(); i++) { +// String address = poisData.getJSONObject(i).getString("address"); +// if (address.contains(searchKey.toString())) { +// resultMap.put("riskLevel", "A"); +// resultModel.setResult(resultMap); +// mav.setModel(resultModel); +// return mav; +// } else { +// continue; +// } +// } + String city = searchAddress.getString("city"); + List poiList = this.findPois(city, searchKey.toString()); + if (CollectionUtils.isNotEmpty(poiList)) { + resultMap.put("riskLevel", "A"); + } else { + resultMap.put("riskLevel", "B"); } - resultMap.put("riskLevel", "B"); resultModel.setResult(resultMap); mav.setModel(resultModel); return mav; @@ -3352,6 +3366,44 @@ public class AddressController { return mav; } + private List findPois(String city, String address) { + JSONObject reqBody = new JSONObject(); + reqBody.put("from", 0); + reqBody.put("size", 1); + JSONObject query = new JSONObject(); + JSONObject bool = new JSONObject(); + JSONArray must = new JSONArray(); + must.add(addTerm("city", city)); + must.add(addMatchPhrase("address", address)); + bool.put("must", must); + query.put("bool", bool); + reqBody.put("query", query); + Base64 token = new Base64(); + String authenticationEncoding = token.encodeAsString((""+ES_USER+":"+ES_PASSWORD+"").getBytes()); + HttpRequest request = new HttpRequest(); + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Basic "+authenticationEncoding+""); + try { + String response = request.postWithBody("http://192.168.0.45:9200/association/_search", reqBody, headers, "UTF-8"); + JSONObject json = JSONObject.fromObject(response); + if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){ + return Collections.emptyList(); + } + JSONArray hits = json.getJSONObject("hits").getJSONArray("hits"); + List results = new ArrayList(); + for(int i=0 ;i < hits.size(); i++){ + JSONObject jsonObject = hits.getJSONObject(i); + results.add(jsonObject); + } + return results; + + } catch (HttpRequestException e) { + e.printStackTrace(); + } + return Collections.emptyList(); + } + public void setAddressService(AddressService addressService) { this.addressService = addressService; } From 28e2ca8a06dfd8fe2ab73da8af54c47f56f6f297 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 29 May 2023 11:18:38 +0800 Subject: [PATCH 37/50] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=B5=99=E5=95=86?= =?UTF-8?q?=E9=A3=8E=E9=99=A9=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/AddressController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index ad2c3adbd..0a2cc8803 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -87,7 +87,7 @@ import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -//@Authority(role = Role.USER) +@Authority(role = Role.USER) public class AddressController { private AddressService addressService; @@ -3287,7 +3287,7 @@ public class AddressController { return mav; } - //@Authority(role = Role.USER) + @Authority(role = Role.USER) public Object riskAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); // if (StringUtils.isEmpty(model.getCity())) { From 572d628aa057e660846e6ed8ace79c9e507e98fd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 29 May 2023 13:49:20 +0800 Subject: [PATCH 38/50] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=BC=80=E4=B8=9A?= =?UTF-8?q?=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/webapp/district2/edit.ftl | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl index c0035175a..882fa8ed6 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district2/edit.ftl @@ -518,18 +518,18 @@ - - - <#if markMap?? && markMap["live_date"]?? && markMap["live_date"] == 1> - - <#else> - - - 寮�涓氭椂闂� - - - - +<#-- --> +<#-- --> +<#-- <#if markMap?? && markMap["live_date"]?? && markMap["live_date"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 寮�涓氭椂闂�--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> <#if markMap?? && markMap["finish_building_date"]?? && markMap["finish_building_date"] == 1> From 7ac13fd481720420a7c483951d4b9e3e5d81e9f1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 29 May 2023 14:32:56 +0800 Subject: [PATCH 39/50] =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/PropertyController.java | 10 +++++----- .../open/controllers/weixin/DistrictController.java | 2 -- .../fangjia-open/src/main/resources/urlrewrite2.0.xml | 6 ++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 5e3c7c3d7..e44529895 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -3668,11 +3668,11 @@ public class PropertyController { } else { b.setDirection(""); } - if (house.getPics() != null && !house.getPics().isEmpty()) { - b.setPic(transformPicsToFangJia(house.getPics().get(house.getPics().size() / 2))); - } else { - b.setPic(""); - } +// if (house.getPics() != null && !house.getPics().isEmpty()) { +// b.setPic(transformPicsToFangJia(house.getPics().get(house.getPics().size() / 2))); +// } else { +// b.setPic(""); +// } if (house.getPublishDate() != null) { if (new Date().before(house.getPublishDate())) { house.setPublishDate(house.getModificationDate()); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java index 674834e68..6ad4b19bc 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java @@ -141,7 +141,6 @@ public class DistrictController { } public Object districtDetailAction(DistrictModel districtModel,HttpServletRequest request,HttpServletResponse response) { - System.out.println("小区详情coming"); //id和name不能同时为空 if(StringUtils.isBlank(districtModel.getId()) && (StringUtils.isBlank(districtModel.getCityName()) || StringUtils.isBlank(districtModel.getName()))) { districtModel.setMsg("参数错误"); @@ -150,7 +149,6 @@ public class DistrictController { District district = null; if(StringUtils.isNotBlank(districtModel.getId())) { - System.out.println("小区详情1"); district = districtService.findById(districtModel.getId()); if(district != null) { diff --git a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml index c17530d88..22c9f3057 100644 --- a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml +++ b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml @@ -43,5 +43,11 @@ /address/fiveDivisionStructure + + + /property/detail + + + \ No newline at end of file From 78d9968e302e133995c17c59fa77f08776fbd0d4 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 31 May 2023 10:04:56 +0800 Subject: [PATCH 40/50] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=81=94=E6=83=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddrSvcController.java | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index d17fb74f3..95d5b2ad4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -11,7 +11,6 @@ import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.open.util.CommonDtoUtils; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; -import org.springframework.stereotype.Controller; import java.util.HashMap; import java.util.Map; @@ -44,20 +43,20 @@ public class AddrSvcController { Map params = new HashMap(); params.put("city", model.getCity()); params.put("address", model.getAddress()); - try{ + try { json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure/v2", params, "UTF-8"); if (json != null) { int code = json.getInt("status"); String msg = json.getString("msg"); BasicModel basicModel = new BasicModel(code, msg); - if(json.containsKey("result")){ + if (json.containsKey("result")) { basicModel.setResult(CommonDtoUtils.transformV(StructureDto.class, json.get("result"))); } mav.setModel(basicModel); } else { mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } - } catch(Exception e){ + } catch (Exception e) { e.printStackTrace(); mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } @@ -83,20 +82,20 @@ public class AddrSvcController { Map params = new HashMap(); params.put("city", model.getCity()); params.put("address", model.getAddress()); - try{ + try { json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8"); if (json != null) { int code = json.getInt("status"); String msg = json.getString("msg"); BasicModel basicModel = new BasicModel(code, msg); - if(json.containsKey("result")){ + if (json.containsKey("result")) { basicModel.setResult(CommonDtoUtils.transformV(DistrictIdentifyDto.class, json.get("result"))); } mav.setModel(basicModel); } else { mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } - } catch(Exception e){ + } catch (Exception e) { e.printStackTrace(); mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } @@ -121,20 +120,20 @@ public class AddrSvcController { Map params = new HashMap(); params.put("city", model.getCity()); params.put("address", model.getAddress()); - try{ + try { json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8"); if (json != null) { int code = json.getInt("status"); String msg = json.getString("msg"); BasicModel basicModel = new BasicModel(code, msg); - if(json.containsKey("result")){ + if (json.containsKey("result")) { basicModel.setResult(CommonDtoUtils.transformV(StructureAndCompleteDto.class, json.get("result"))); } mav.setModel(basicModel); } else { mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } - } catch(Exception e){ + } catch (Exception e) { e.printStackTrace(); mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } @@ -157,20 +156,20 @@ public class AddrSvcController { Map params = new HashMap(); params.put("city", model.getCity()); params.put("address", model.getAddress()); - try{ + try { json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/addrVerify", params, "UTF-8"); if (json != null) { int code = json.getInt("status"); String msg = json.getString("msg"); BasicModel basicModel = new BasicModel(code, msg); - if(json.containsKey("result")){ + if (json.containsKey("result")) { basicModel.setResult(json.get("result")); } mav.setModel(basicModel); } else { mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } - } catch(Exception e){ + } catch (Exception e) { e.printStackTrace(); mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } @@ -195,20 +194,20 @@ public class AddrSvcController { params.put("address", model.getAddress()); params.put("category", model.getCategory()); params.put("district", model.getSuggestDistrict()); - try{ + try { json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/match", params, "UTF-8"); if (json != null) { int code = json.getInt("status"); String msg = json.getString("msg"); BasicModel basicModel = new BasicModel(code, msg); - if(json.containsKey("result")){ + if (json.containsKey("result")) { basicModel.setResult(CommonDtoUtils.transformV(MatchedBuildingDto.class, json.get("result"))); } mav.setModel(basicModel); } else { mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } - } catch(Exception e){ + } catch (Exception e) { e.printStackTrace(); mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } @@ -239,21 +238,24 @@ public class AddrSvcController { params.put("region", model.getRegion()); params.put("keywords", model.getKeyword()); params.put("page", "0"); - params.put("size", String.valueOf(model.getMax() == 0 ? 20 : (model.getMax() > 100 ? 100 : model.getMax()) )); - try{ + params.put("size", String.valueOf(model.getMax() == 0 ? 20 : (model.getMax() > 100 ? 100 : model.getMax()))); + try { json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/addressAssociation2", params, "UTF-8"); if (json != null) { int code = json.getInt("status"); String msg = json.getString("msg"); BasicModel basicModel = new BasicModel(code, msg); - if(json.containsKey("result")){ - basicModel.setResult(CommonDtoUtils.transformListV(DistrictDto.class, (JSONObject.fromObject(json.get("result"))).getJSONArray("content"))); + if (json.containsKey("result")) { + JSONObject result = (JSONObject.fromObject(json.get("result"))); + if (result.getInt("totalElements") > 0) { + basicModel.setResult(CommonDtoUtils.transformListV(DistrictDto.class, result.getJSONArray("content"))); + } } mav.setModel(basicModel); } else { mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } - } catch(Exception e){ + } catch (Exception e) { e.printStackTrace(); mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); } From 7c29e5f2d16f7ce6446828bbe04415de11a0022a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 31 May 2023 14:17:14 +0800 Subject: [PATCH 41/50] =?UTF-8?q?=E6=88=BF=E6=BA=90=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/webapp/seawater/houseList.ftl | 14 ++++ .../dal/seawater/dao/SeawaterDaoImpl.java | 25 +++++++ .../fangjia/dal/seawater/model/Seawater.java | 70 +++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl b/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl index 38771bc5d..6955b465c 100644 --- a/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl @@ -158,6 +158,13 @@ 鏉垮潡 瑁呬慨 鏈濆悜 + 姊� + 鎴� + 鎴峰瀷缁撴瀯 + 浜ゆ槗鏉冨睘 + 鎴挎潈鎵�灞� +<#-- 鎴垮眿骞撮檺--> + 閰嶅鐢垫 鐗╀笟绫诲瀷 鏉ユ簮 鐘舵�� @@ -200,6 +207,13 @@ ${it.block!} ${it.fitment!} ${it.direction!} + ${it.lift!} + ${it.liftHousehold!} + ${it.roomStructure!} + ${it.tradeProperty!} + ${it.propertyOwnership!} +<#-- ${it.housingAge!}--> + ${it.elevator!} ${it.estateType!} <#if it.url??> diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index adb32083b..ab018c2ac 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -347,6 +347,31 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(source.containsKey("status")){ seawater.setStatus(source.getInt("status")); } + + if(source.containsKey("lift")){ + seawater.setLift(source.getInt("lift")); + } + + if(source.containsKey("lift_household")){ + seawater.setLiftHousehold(source.getInt("lift_household")); + } + + if(source.containsKey("room_structure")){ + seawater.setRoomStructure(source.getString("room_structure")); + } + + if(source.containsKey("trade_property")){ + seawater.setTradeProperty(source.getString("trade_property")); + } + if(source.containsKey("property_ownership")){ + seawater.setPropertyOwnership(source.getString("property_ownership")); + } + if(source.containsKey("housing_age")){ + seawater.setHousingAge(source.getString("housing_age")); + } + if(source.containsKey("elevator")){ + seawater.setElevator(source.getString("elevator")); + } seawaterList.add(seawater); } return seawaterList; diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java index 1af563193..15176face 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java @@ -62,6 +62,13 @@ public class Seawater implements SeawaterEntity { private String purpose; private String estateType; private String buildType; + private Integer lift; + private Integer liftHousehold; + private String roomStructure;//户型结构 + private String tradeProperty; //交易权属 + private String propertyOwnership; //房权所属 非共有 + private String housingAge; //房屋年限 满二年 + private String elevator; //配备电梯 有/无 @Id @Column(name = "_id") @@ -325,4 +332,67 @@ public class Seawater implements SeawaterEntity { public void setBuildType(String buildType) { this.buildType = buildType; } + + @Column(name = "lift") + public Integer getLift() { + return lift; + } + + public void setLift(Integer lift) { + this.lift = lift; + } + + @Column(name = "lift_household") + public Integer getLiftHousehold() { + return liftHousehold; + } + + public void setLiftHousehold(Integer liftHousehold) { + this.liftHousehold = liftHousehold; + } + + @Column(name = "room_structure") + public String getRoomStructure() { + return roomStructure; + } + + public void setRoomStructure(String roomStructure) { + this.roomStructure = roomStructure; + } + + @Column(name = "trade_property") + public String getTradeProperty() { + return tradeProperty; + } + + public void setTradeProperty(String tradeProperty) { + this.tradeProperty = tradeProperty; + } + + @Column(name = "property_ownership") + public String getPropertyOwnership() { + return propertyOwnership; + } + + public void setPropertyOwnership(String propertyOwnership) { + this.propertyOwnership = propertyOwnership; + } + + @Column(name = "housing_age") + public String getHousingAge() { + return housingAge; + } + + public void setHousingAge(String housingAge) { + this.housingAge = housingAge; + } + + @Column(name = "elevator") + public String getElevator() { + return elevator; + } + + public void setElevator(String elevator) { + this.elevator = elevator; + } } From 10fe787116c2f9b7a37584c6bae42afaee47de41 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 31 May 2023 19:37:06 +0800 Subject: [PATCH 42/50] =?UTF-8?q?=E6=B8=A4=E6=B5=B7=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 85 ++++++------------- .../com/fangjia/open/util/BHBankUtils.java | 18 +++- 2 files changed, 45 insertions(+), 58 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index af890d880..e11dc5cbb 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1,41 +1,6 @@ package com.fangjia.open.controllers; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.validation.Valid; - import com.fangjia.core.util.*; -import net.sf.json.JSONArray; -import net.sf.json.JSONObject; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.TermQuery; -import org.apache.poi.hssf.usermodel.HSSFSheet; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fangjia.dal.QueryParam; import com.fangjia.dal.SdQuery; import com.fangjia.dal.SrQuery; @@ -54,32 +19,14 @@ import com.fangjia.open.OpenConfig; import com.fangjia.open.dto.UniversalDto; import com.fangjia.open.interceptors.Authority; import com.fangjia.open.interceptors.Role; -import com.fangjia.open.model.AddesssMatchModel; -import com.fangjia.open.model.AddressModel; -import com.fangjia.open.model.BasicModel; -import com.fangjia.open.model.RateModel; -import com.fangjia.open.model.RestfulModelAndView; +import com.fangjia.open.model.*; import com.fangjia.service.address.AddressParser; import com.fangjia.service.address.AddressParserService; import com.fangjia.service.address.AddressParserUtil; -import com.fangjia.service.address.mode.Address; -import com.fangjia.service.address.mode.AddressMatchMetaData; -import com.fangjia.service.address.mode.AddressParseResult; -import com.fangjia.service.address.mode.AddressParserGoal; -import com.fangjia.service.address.mode.AddressRate; -import com.fangjia.service.address.mode.Building; -import com.fangjia.service.address.mode.BuildingType; -import com.fangjia.service.address.mode.MatchParams; -import com.fangjia.service.address.mode.MatchText; -import com.fangjia.service.address.mode.PreciseLevel; +import com.fangjia.service.address.mode.*; import com.fangjia.service.api.AddressMatchProxyService; import com.fangjia.service.api.BaiduApiService; -import com.fangjia.service.bo.AddressCheckBo; -import com.fangjia.service.bo.AddressCheckTruth; -import com.fangjia.service.bo.AroundConsumeGradeBo; -import com.fangjia.service.bo.CheckMatchDistrictCorrectParam; -import com.fangjia.service.bo.CityParseBo; -import com.fangjia.service.bo.MatchDistrictResult; +import com.fangjia.service.bo.*; import com.fangjia.service.datastore.CommDataService; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.datastore.EvaluationService; @@ -89,7 +36,31 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService; import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -import org.sothis.web.mvc.annotation.Param; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.poi.hssf.usermodel.HSSFSheet; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.validation.Valid; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.util.*; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @Authority(role = Role.USER) public class AddressController { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 1b7652515..21ee76954 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -45,6 +45,18 @@ public class BHBankUtils { put("22","崇明"); }}; + public static Map houseTypeMap = new HashMap(){{ + put("8","公寓"); + put("4","独立别墅"); + put("5","双拼别墅"); + put("6","联排别墅"); + put("7","叠加别墅"); + put("9","联列住宅"); + put("10","花园住宅"); + put("11","旧式里弄2"); + put("0","其它"); + }}; + public static List getDistrictByBhbank(String address){ List list = new ArrayList(); try { @@ -116,6 +128,10 @@ public class BHBankUtils { if(StringUtils.isNotBlank(ev.getRoomNum())){ params.put("houseNo",ev.getRoomNum()); } + + if(StringUtils.isNotBlank(String.valueOf(ev.getBuildType()))){ + params.put("houseType",houseTypeMap.get(String.valueOf(ev.getBuildType()))); + } if(ev.getHouseArea()>0){ if(ev.getBasementArea()!=null && ev.getBasementArea()>0){ double area = ev.getHouseArea() - ev.getBasementArea(); @@ -141,7 +157,7 @@ public class BHBankUtils { params.put("groupName", usersDetail.getOrgName()); - List lists = Arrays.asList("queryFullAddr", "preCode", "preName", "countryCode", "buildName", "unitCode","floor","totalFloor","houseNo","groundArea","undergroundArea","orientName","buildYear","dealPriceTotal","specialNo","userName","userPhone","userMail","groupName"); + List lists = Arrays.asList("queryFullAddr", "preCode", "preName", "countryCode", "buildName", "unitCode","floor","totalFloor","houseNo","groundArea","undergroundArea","orientName","buildYear","dealPriceTotal","specialNo","userName","userPhone","userMail","groupName","houseType"); String s = ""; for(String key : lists){ From 45b5d44930d73c1fd929060eca0ef37ef0cff0ac Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 1 Jun 2023 09:45:33 +0800 Subject: [PATCH 43/50] update --- .../src/main/java/com/fangjia/open/util/BHBankUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 21ee76954..77f54b39e 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -45,6 +45,7 @@ public class BHBankUtils { put("22","崇明"); }}; + //房屋类型传参 public static Map houseTypeMap = new HashMap(){{ put("8","公寓"); put("4","独立别墅"); From 01cb796cfaa343506a463b88732a1fe3a905e13b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 1 Jun 2023 21:16:57 +0800 Subject: [PATCH 44/50] =?UTF-8?q?=E5=8A=A0=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/HouseController.java | 4 ++-- .../java/com/fangjia/open/controllers/JnBankController.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index 5f2fddf88..1efa12d0f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -208,7 +208,7 @@ public class HouseController { } - @Authority(role = Role.NONE) + @Authority(role = Role.USER) public Object similarAction(@Valid CollateralSampleQuery collateralSampleQuery){ RestfulModelAndView mav = new RestfulModelAndView(); @@ -253,7 +253,7 @@ public class HouseController { /**商铺相似房源接口*/ - @Authority(role = Role.NONE) + @Authority(role = Role.USER) public Object shopSimilarAction(@Valid ShopCollateralSampleQuery query){ RestfulModelAndView mav = new RestfulModelAndView(); ShopCollateralSampleQueryBean queryBean = new ShopCollateralSampleQueryBean(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java index 998b4664e..2c7a22187 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java @@ -39,7 +39,7 @@ public class JnBankController { private static final Pattern STAGE_PATTERN = Pattern.compile(".*[\\w东南北中西甲乙丙丁一二三四五六七八九十百千\\d]+(期|区)$"); - @Authority(role = Role.NONE) + @Authority(role = Role.USER) public Object searchAction(@Valid JNBankSearchModel model){ RestfulModelAndView mav = new RestfulModelAndView(); From 7bd916a214d18218a6542f7c39f4be73ccb42463 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 9 Jun 2023 11:07:51 +0800 Subject: [PATCH 45/50] update --- .../script/ana/DistrictStatRankBatch.groovy | 102 ++++++- .../BigdataCommunityBatch.groovy | 8 +- .../ccbevaluation/BigdataDealBatch.groovy | 89 +++--- .../DealAndThirdPriceDataProcess.groovy | 44 ++- .../ExportOracleAllDataTest.groovy | 6 +- .../ccbevaluation/FjwPricePkCcbPrice.groovy | 22 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 8 +- .../ronghe/ExportDistrictRongHe.groovy | 2 +- .../script/chongqing/FumingTest.groovy | 261 ++++++++++++++++++ .../script/chongqing/MingShenPocTest.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 32 ++- .../fuzhou/DistrictInfoExportBatch.groovy | 26 +- .../fuzhou/DistrictSellStatBatch.groovy | 14 +- .../script/fuzhou/FzDataResultStat.groovy | 2 +- .../com/fangjia/script/fuzhou/Test.groovy | 98 +++++-- .../fangjia/script/gj/AddAvgpriceBatch.groovy | 7 +- .../fangjia/script/gj/CityStatExport.groovy | 15 +- .../com/fangjia/script/gj/FiledStrand.groovy | 45 +++ .../com/fangjia/script/gj/OpenApiUtils.groovy | 76 ++++- .../com/fangjia/script/gj/TradeMatch.groovy | 49 ++++ .../com/fangjia/script/gj/ZheShengBank.groovy | 61 ++-- .../fangjia/script/gj/data/XiamenStat.groovy | 207 ++++++++++++++ .../gj/district/DistrictFieldAddBatch.groovy | 2 +- .../gj/district/FangjiaClientUtils.groovy | 2 +- .../gj/district/OfficeFieldCleanBatch.groovy | 79 ++++++ .../district/SeaweedDataMatchProcess.groovy | 3 + .../gj/es/FangjiaCorestieLogAnaly.groovy | 17 +- .../fangjia/script/gj/es/HouseAnaly.groovy | 232 ++++++++++++++++ .../gj/location/GaoDeMapSearchTest.groovy | 16 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 94 +++++-- .../script/gj/paimai/DataFormatUtils.groovy | 78 ++++++ .../gj/paimai/LianjiaCrawlByChrome.groovy | 167 ++++++----- .../gj/region/StandardOtherWebRegion.groovy | 36 +++ .../com/fangjia/script/util/DbUtils.groovy | 2 +- 34 files changed, 1669 insertions(+), 237 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy index 6dcae7f53..a11b60d8d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy @@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ServerAreaMap import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * Created by dxy on 2016/12/15. */ @@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch { private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def DISTRICT_STAT_MAP = [:] public void process(Object args) { - def cities = ServerAreaMap.findAllCities() + + + def cities = ["深圳"]//ServerAreaMap.findAllCities() for(String city : cities) { - DISTRICT_STAT_MAP.clear() - Map> regions = RegionData.getAllRegions(city) - regions.each {region, alias-> - processRegionRank(city, region) - } - processCityRank(city) +// DISTRICT_STAT_MAP.clear() +// Map> regions = RegionData.getAllRegions(city) +// regions.each {region, alias-> +// processRegionRank(city, region) +// } +// processCityRank(city) + + processCityRentRank(city) } } @@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch { return ret } + def processCityRentRank(city){ + def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd") + def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth) + def prices = [] + distRentsMap.each{key, price-> + if(!prices.contains(price)){ + prices << price + } + } +// prices = prices.unique() + prices = prices.sort().reverse() + + def totalCount = new AtomicLong() + def keys = distRentsMap.keySet().asList() + final CountDownLatch latch = new CountDownLatch(keys.size()) + +// for(def i=0;i + index ++ + def arrs = key.split("##"); + double percentile = calculatePercentile(prices, price); + + DBObject q = new BasicDBObject("city", arrs[0]) + q.append("region", arrs[1]) + q.append("district_name", arrs[2]) + + DBObject udata = new BasicDBObject() + udata.put("rent_city_rank", (int)percentile); + udata.put("m_date", new Date()) + DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false) + println index +" "+ arrs[2] + } + } + + def upateDistrict(key, distRentsMap, prices){ + def arrs = key.split("##"); + def price = distRentsMap.get(key) + + double percentile = calculatePercentile(prices, price); + + DBObject q = new BasicDBObject("city", arrs[0]) + q.append("region", arrs[1]) + q.append("district_name", arrs[2]) + + DBObject udata = new BasicDBObject() + udata.put("rent_city_rank", (int)percentile); + udata.put("m_date", new Date()) +// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false) + } + + public static int calculatePercentile(def values, def targetValue) { + // 找出数字在List中的排名 + def rank = values.indexOf(targetValue) + 1 + // 计算百分位数 + def percentile = (int)((rank / values.size()) * 100) + return percentile; + } static main(args) { runBatch(new DistrictStatRankBatch(), args) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c52b21fcb..17860e977 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) -// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 - def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 +// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -78,8 +78,8 @@ class BigdataCommunityBatch extends BaseBatch{ println "txt 生成->导Oracle" - delPgData() - saveToPgSql(rows) +// delPgData() +// saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 6854ae591..4d289d469 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -66,50 +66,51 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } println "curDate:"+curDate + def allCitys = CommonDatautils.findAllCity() + for(def city : allCitys){ +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } + } +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 // for(def city : citys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } +// index ++ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// // } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 - for(def city : citys){ - index ++ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - - } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成 " + rows.size() - - saveToPgSql(rows) +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成 " + rows.size() +// +// saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{ def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) - def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) + def trades = CcbCommonUtils.findTradesByMonth(city, startDate) final CountDownLatch latch = new CountDownLatch(trades.size()) def final AtomicInteger count_ = new AtomicInteger(0) @@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{ rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address); } } - if(rMap){ + if(rMap && rMap.region==region){ CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index cdad2a607..1e818b3ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -28,9 +28,11 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); - task.deleteThirdPartyData() +// task.deleteThirdPartyData()//删除第三方 // task.delDealByPe() // task.delDealData() + + task.exportThirdParty() } def delDealByPe(){ @@ -372,15 +374,29 @@ class DealPriceDataProcess { return map } + def exportThirdParty(){ + def citys = CcbCommonUtils.citys_all + def results = [] + results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE'] + for(def city : citys) { + def rets = findPeGteThirdParty3(city, 202305) + for(def map : rets){ + results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe] + } + println city +" "+ rets.size() + } + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方") + } def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202303 + def month = 202304 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202304) + def rets = findPeGteThirdParty2(city, 202305) def total = 0 println city +" "+rets.size() if(rets){ @@ -431,6 +447,28 @@ class DealPriceDataProcess { return rets } + //城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE + + def findPeGteThirdParty3(cityName, yearmonth){ + def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15" + def rets = [] + pgsqlInstance.eachRow(sql,{r-> + def map = [:] + map.cityid = r.urbn_id + map.city = r.urbn_nm + map.data_id = r.gujia_fsid + map.region = r.gujia_rgon_nm + map.comm_id = r.gujia_comm_id + map.district_name = r.gujia_comm_nm + map.type = r.gujia_bds_clc_tp + map.clc_price = r.gujia_bds_avprc + map.price = r.gujiabds_to_mkprc_mny + map.ppe = r.gujia_mns_rto + rets << map + }); + return rets + } + def delDealData(){ ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 4df84ea86..8c765a987 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -79,9 +79,9 @@ class ExportOracleAllDataTest { // exportCommCaseDetailGj()// // // ok文件 - exportInOk() - SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); +// exportInOk() +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 5c2e8c26c..60e475036 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -33,7 +33,7 @@ class FjwPricePkCcbPrice { def task = new FjwPricePkCcbPrice() // task.run() // task.updatePrice() -// task.addFjwPrice() + task.addFjwPrice() // task.addCcbPrice()//oracle // task.addCcbPriceFromGuided() @@ -42,7 +42,7 @@ class FjwPricePkCcbPrice { // task.exportFjwPriceAndCcbPrice() // task.addNewDistPrice() - task.addCcbPrice2()//按id查 +// task.addCcbPrice2()//按id查 // task.addCcbPrice3()//按city,region,name查 // task.standRegion() // task.addGuidPrice() @@ -217,17 +217,19 @@ class FjwPricePkCcbPrice { } def addFjwPrice(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d-> - if(!d['K']){ - def dist = CommonDatautils.findDistrictById(d['C']) + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d-> + if(!d['N']){ + def dist = CommonDatautils.findDistrictById(d['E']) if(dist){ - def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd")) - if(dbo){ - d['K'] = dbo.avg_price - } + d['N'] = dist.region +// def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd")) +// if(dbo){ +// d['K'] = dbo.avg_price +// } + + } } - }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 953a53a6b..91fb02cdc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -83,9 +83,9 @@ class GuiDingPriceBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); saveToPgSql(rows) - - //更新静态数据表 - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") +// +// //更新静态数据表 +// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 852f3783a..ae9f4f058 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy new file mode 100644 index 000000000..a009eb954 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy @@ -0,0 +1,261 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql + +class FumingTest { + + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + + static main(args) { + def task = new FumingTest(); + task.run(); + } + + def run() { + def path = "D:\\数据\\富民\\富民测试500-v7.xlsx" +// match(path) +// addPrice(path) +// addThirdPrice(path) +// addLastPrice(path) + addFacLevel(path) + } + + + def addFacLevel(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['K'] && row['I']){ +// def obj = findCommunitySummary(row['C']) +// if(obj){ +// row['K'] = obj.consumption_rating +// }else{ + row['K'] = getNearByDistrictLevel(row['C'], row['I']) +// } + } + }) + } + + + /** + * 一公里小区数量 + */ + def getNearByDistrictLevel(def id, def price){ + BasicDBObject district = DbUtils.seaweed.findById(id); + def level = null + if(district){ + List list = DbUtils.findAroundDistrict(district); + println id +" 共:"+list.size()+" "+ price + if(list!=null&&list.size()>0){ + for(def obj : list){ + def summary = findCommunitySummary(obj._id.toString()) + if(summary && summary.avg_price && summary.consumption_rating){ + def pre = java.lang.Math.abs((summary.avg_price - price)*100/price) + println summary.name +" "+pre +" "+ summary.avg_price + if(pre <= 10){ + level = summary.consumption_rating + break; + } + } + } + } + } + return level; + } + + + def addLastPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Z'] ){ + if(row['W']){ + row['Z'] = row['W'] + }else if(row['R']){ + row['Z'] = row['R'] + }else if(row['I']){ + row['Z'] = row['I'] + } + + if(row['Z']){ + row['J'] = (int)(row['Z']*row['B'].toString().toDouble()) + } + } + }) + } + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['N'] ){ + def city = row['C'] + def address = row['A'] + def region = row['D'] + def name = row['G'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) + if(jsonObject != null && jsonObject.precise!=-1) { + row['N'] = jsonObject.get("id") + row['O'] = jsonObject.get("city") + row['P'] = jsonObject.get("district") + row['Q'] = jsonObject.get("name") + row['R'] = jsonObject.get("propertyType") + row['S'] = jsonObject.get("address") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['C'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + row['D'] = dist.city + row['E'] = dist.region + row['F'] = dist.name + row['G'] = dist.estate_type2 + row['H'] = dist.address +// row['Q'] = dist.name + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['I'] = priceObj.avg_price + row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble()) + row['R'] = priceObj.avg_price_2b + } + } + } + }) + } + + + def addThirdPrice(path){ +// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连'] + def allMap = [:] +// for(def c : citys){ +// def cityMap = findThirdPartyPrice(c) +// allMap << cityMap +// } + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F']) + def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['L'] = priceMap.get("lianjia_esf_price") + } + + if(priceMap.containsKey("beike_esf_price")){ + row['M'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("anjuke_esf_price")){ + row['N'] = priceMap.get("anjuke_esf_price") + } + } + def map = findAvgPrice(row['C'], cityCodeMap.get(row['D'])) + if(map){ + row['O'] = map.avg_price + row['P'] = map.credit_value + } + }) + } + + + + def addCcbPrice2(){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d -> + if(d['C']){ + def map = findAvgPrice(d['C'], cityCodeMap.get(d['E'])) + if(map){ + d['K'] = map.avg_price + d['L'] = map.calc_type +// d['AI'] = map.credit_value + d['M'] = map.month + } + } + }) + } + + def findThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + } + return map + } + + def findAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + + + def findCommunitySummary(id){ + def obj = null + def sql = "select * from community_summary where id='${id}'" + ElasticUtils.executeSqlQuery(sql, { + obj = it + }) + return obj + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy index 112de372f..50db279aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy @@ -26,7 +26,7 @@ class MingShenPocTest { def run(){ def path = "D:\\数据\\民生\\poc测试.xlsx" //9级标准化 - //structureV2(path)//针对海南poc地址数据此接口不合适使用 + structureV2(path)//针对海南poc地址数据此接口不合适使用 //省市区拆分 // ner(path)//城市,区域带市带区 // match(path) @@ -34,7 +34,7 @@ class MingShenPocTest { // addLatLng(path) // addAroundPrice(path) // addAuthor(path) - addHouse(path) +// addHouse(path) } def structureV2(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 543973cfa..6c633d4f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -73,6 +73,8 @@ class CommonDatautils { public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl) public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo); + public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -230,6 +232,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); + q.put("name", "长滩美墅三期") DBObject keys = new BasicDBObject(); if(fields){ @@ -781,7 +784,7 @@ class CommonDatautils { public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); q.put("region", new BasicDBObject('$in', regions)) @@ -3418,11 +3421,27 @@ class CommonDatautils { dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def sqmetreRentMap = [:] for(def d : dbcursor){ - sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) +// sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) + sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price) } return sqmetreRentMap; } + + static def findAllSqmetreRentPrices(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("cat","district") + q.put("weekend", weekend) + def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000) + dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def prices = [] + for(def d : dbcursor){ + prices << d.avg_price + } + return prices; + } + static def findNewSqmetreRent(def city, def region, def name){ def q = new BasicDBObject() q.put("city", city) @@ -3490,5 +3509,14 @@ class CommonDatautils { } return obj } + + static def findDistrictAnalyStat(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q) + return analyData + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index ea6d2717b..464ac2301 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -82,20 +82,21 @@ class DistrictInfoExportBatch extends BaseBatch{ } } def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); - for(def dist : removeDists){ - def key = dist.city +"##"+dist.region+"##"+dist.name - if(!distKeys.contains(key)){ - distKeys << key - districts << dist - } - } +// for(def dist : removeDists){ +// def key = dist.city +"##"+dist.region+"##"+dist.name +// if(!distKeys.contains(key)){ +// distKeys << key +// districts << dist +// } +// } def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) def startDate = DateUtils.getFirstDayOfMonth(lastDate); def fzAddrMap = getFzAddress(); //因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据 - saveOrUpdateFzDist(districts) +// saveOrUpdateFzDist(districts) + remarkRemoveDist(removeDists) def fzDistricts = getFzDistricts(city) for(def rMap : fzDistricts){ @@ -185,6 +186,15 @@ class DistrictInfoExportBatch extends BaseBatch{ } + def remarkRemoveDist(removeDists){ + for(def rd : removeDists){ + def udate = new BasicDBObject() + udate.put("status",1) + udate.put("m_date", new Date()) + FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate)); + } + } + def getFzDistricts(def city){ def q = new BasicDBObject() q.put("city", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 96ed55fcb..dc389ffe5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch { if(weekendPriceObj.value_type!=1){ if(dist.city == "福州"){ if(Math.abs(mom)*100>3){ - def randMarkup = nextDouble(0, 3) - if(mom < 0){ - mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) - }else{ - mom = DataUtil.roundDouble(randMarkup/100, 4) + if(dist.region != "马尾"){ + def randMarkup = nextDouble(0, 3) + if(mom < 0){ + mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) + }else{ + mom = DataUtil.roundDouble(randMarkup/100, 4) + } + curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) } - curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) } }else{ if(Math.abs(mom)*100>10){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 917ddfc5e..2da506d11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -25,7 +25,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); - task.regionPriceTrendRun() +// task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 47a3b331f..9cea6cf74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.fuzhou +import com.fangjia.core.RegionData import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils @@ -11,26 +12,91 @@ import com.mongodb.DBCollection import org.apache.commons.io.FileUtils import org.bson.types.ObjectId +DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") -DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") -def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") -def rets = [] -for(def d : datas){ - def arrs = d.split("@") - rets << ["commId":arrs[0], "avg_price":arrs[4]] -} - -for(def r : rets){ - if(r.avg_price){ - def udata = new BasicDBObject() - udata.put("date", 202210) - udata.put("avg", r.avg_price.toInteger()) - DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) - }else{ - println r.commId +" 不存在" +ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> + def city = RegionData.formatCity("余姚") + d['D'] = city + if(city){ + d['E'] = RegionData.formatRegion(city, d['C']) } + +// if(!d['D']){ +// def citys = CommonDatautils.getCityByRegion(d['A']) +// if(citys && citys.size()==1){ +// d['F'] = citys.get(0) +// } +// } +}) + +//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1) +//def m = [:] +//for(def d : datas){ +// m.put(d['B'], d) +//} + +//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d-> +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +// if(dist){ +// def ret = m.get(dist._id.toString()) +// if(ret){ +// d['E'] = ret['J'] +// d['F'] = ret['N'] +// } + +// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS) +// if(dbo){ +// if(dbo.beike_esf_price){ +// d['G'] = dbo.beike_esf_price +// }else if(dbo.beike_esf_price){ +// d['G'] = dbo.beike_esf_price +// } +// +// if(dbo.maitian_esf_price){ +// d['H'] = dbo.maitian_esf_price +// } +// } +// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd")) +// if(dbo){ +// d['K'] = dbo.avg_price +// d['L'] = dbo.value_type +// } +// +// } +//}) + + + + + +def findThirdPrice(city, region, name, AVERAGE_STATISTICS){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", 202304) + return AVERAGE_STATISTICS.findOne(q) } +//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + +//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") +//def rets = [] +//for(def d : datas){ +// def arrs = d.split("@") +// rets << ["commId":arrs[0], "avg_price":arrs[4]] +//} +// +//for(def r : rets){ +// if(r.avg_price){ +// def udata = new BasicDBObject() +// udata.put("date", 202210) +// udata.put("avg", r.avg_price.toInteger()) +// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) +// }else{ +// println r.commId +" 不存在" +// } +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 8ccb30a9a..a57d7551b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -29,8 +29,11 @@ class AddAvgpriceBatch { } def test(){ - //发重算均价命令 - addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd')) + def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"]) + for(def dist : districts){ + //发重算均价命令 + addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd')) + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index d6432197b..5cae7c343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -96,17 +96,28 @@ class CityStatExport { } def regionStat(){ - def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州'] def ret = [] ret << ["城市","区域","小区总量"] for(def city : citys){ + def regions = CommonDatautils.getRegionListByCity(city) for(def r : regions){ + def row = [] + row << city + row << r def districts = CommonDatautils.findDistricts(city, r, null) println city +" "+ r +" "+ districts.size() + + row << CommonDatautils.queryCityRegionPrice(city, r, null) + def obj = CommonDatautils.findRegionPriceRate(city, r) + row << obj.price_coverage*100 + row << districts.size() + + ret << row } } - ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县") + ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1b7d3b371..df4d9454e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -272,6 +272,45 @@ class FiledStrand { return null } + static def processName(name){ + name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期') + if (name.contains('(') && name.contains(')')){ + def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1) + name = name.replace(rep_str, '') + } + + + if (name.contains('(') && name.contains(')')){ + def rep_str = name.subsring(name.index('('),name.rfind(')')+1) + name = name.replace(rep_str, '') + } + + + if (name.find('【')!=-1 && name.find('】')!=-1){ + def rep_str = name[name.index('【'):name.rfind('】')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('【')!=-1 && name.find('】')!=-1){ + def rep_str = name[name.index('【'):name.rfind('】')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('(')!=-1 && name.find(')')!=-1){ + def rep_str = name[name.index('('):name.rfind(')')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('「')!=-1 && name.find('」')!=-1){ + def rep_str = name[name.index('「'):name.rfind('」')+1] + name = name.replace(rep_str, '') + } + return name + } + static def processCarSite(car_site_count){ def v = getValue(car_site_count.toString()); if(v){ @@ -579,6 +618,12 @@ class FiledStrand { }else{ return null; } + }else if(estateType2=="写字楼"){ + if(estateCharege>=0.35 && estateCharege<=60){ + return estateCharege; + }else{ + return null; + } } }catch (Exception e){ println "estateCharege 转不了 "+estateCharege diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index e4e5016ec..7555e8915 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -567,7 +567,7 @@ class OpenApiUtils { } static def financialRating(city, address){ - def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3" + def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" String content = new HttpRequest().get(url, "utf-8"); if(content){ def doc = new JsonSlurper().parseText(content) @@ -580,5 +580,79 @@ class OpenApiUtils { } + //源地址的完整性进行验证 + static def addrVerify(city, address){ + def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + // 省、市、区、街道、社区居委、POI补全 + static def complete(city, address){ + def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + //五级(省/市/区/街道/社区居委)解析、补全 + static def division(city, address){ + def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + //识别地址中省、市、区、道路、POI同时进行地址验证 + static def structure(city, address){ + def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + + /** + * 招商银行测试 + * @param city + * @param address + * @return + */ + static def cmbTest(city, address){ + def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy new file mode 100644 index 000000000..082e2f170 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.gj + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> + if(d['A'] && !d['E']){ +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") +// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") + println d['A'] +" "+d['C'] +// def total = findDatas(d['A'], d['B'], d['C']) +// d['D'] = total + def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) + d['E'] = c + } +}) + + + + +def getAmapLocation(city, region, name){ + String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ d-> + if(d.name.contains(name) && d.type.contains("住宅")){ + total ++ + } + }) + return total; +} + + +def findDatas(def city, def region, def name){ + def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'" + def total = 0 + ElasticUtils.executeSqlQuery(sql, { d-> + total = d.count + }) + return total; +} + +//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d-> +// def region = RegionData.formatCity(d['A']) +// if(!region && d['A']){ +// println d['A'] +// } +//}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy index 73ea03b09..e2eb3f289 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy @@ -76,24 +76,24 @@ MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S -def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") -Calendar calendar = Calendar.getInstance() -calendar.setTime(startDate) -def count = 0 -def hashSet = [] -def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] - -OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> - def result = JSONObject.fromObject(doc.res_map.result) - def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] - data << item - hashSet << result -} - - -//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); -ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") +//def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") +//def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") +//Calendar calendar = Calendar.getInstance() +//calendar.setTime(startDate) +//def count = 0 +//def hashSet = [] +//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] +// +//OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> +// def result = JSONObject.fromObject(doc.res_map.result) +// def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] +// data << item +// hashSet << result +//} +// +// +////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +//ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") //def startDate = DateUtil.parseDate("20221201", "yyyyMMdd") @@ -115,4 +115,27 @@ ExcelUtils.write("D:\\ // // ////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); -//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file +//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") + + + + +def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230524", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]] + +OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) + def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode, + doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber] + data << item + hashSet << result +} + + +//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy new file mode 100644 index 000000000..9eab92fdb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy @@ -0,0 +1,207 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.util.DateUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 按区域统计覆盖率 + */ + +class XiamenStat { + + public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com") + + public static final def DISTRICT_FIELDS_MAP = [ + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' + ] + + static main(args) { + def task = new XiamenStat(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d-> + if(d['D'] && d['D']=="全市"){ + def dbo = CommonDatautils.findCityInfo(d['C'].replace("市","")) + if(dbo){ + def detail = dbo.district_detail + if(detail){ + d['E'] = dbo.existing_district_count//数据量 + d['F'] = detail.township*100// 乡镇街道 + d['G'] = detail.address*100// 地址 + d['H'] = detail.addrAlias*100// 地址别名 + d['I'] = detail.name*100// 名称 + d['J'] = detail.alias*100// 别名 + d['K'] = detail.alias*100// 映射别名 + d['L'] = detail.lat2*100// 经纬度(百度) + d['M'] = detail.lat*100// 经纬度(高德) + d['N'] = detail.estateType2*100// 物业类型 + d['O'] = detail.buildType*100// 建筑类型 + d['P'] = detail.streetCommunity*100// 街道社区 + d['Q'] = detail.committee*100// 居委会 + d['R'] = detail.businessQuarter*100// 商圈 + d['S'] = detail.schools*100// 对口小学 + d['T'] = detail.kindergartens*100// 对口幼儿园 + d['U'] = detail.loudongCount*100// 楼栋总数 + d['V'] = detail.householdCount*100// 户数 + d['W'] = detail.estateCharge*100// 物业费 + d['X'] = detail.carSiteCount*100// 停车位 + d['Y'] = ""// 电梯 + d['Z'] = detail.propertyLife*100// 产权年限 + d['AA'] = detail.houseProperty*100// 产权性质 + d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分) + d['AC'] = detail.landProperty*100// 小区性质 + d['AD'] = detail.finishBuildingDate*100// 竣工时间 + d['AE'] = detail.openDate*100// 交房时间 + d['AF'] = detail.allArea*100// 占地面积 + d['AG'] = detail.buildArea*100// 建筑面积 + d['AH'] = detail.cubagePercent*100// 容积率 + d['AI'] = detail.virescencePercent*100// 绿化率 + d['AJ'] = detail.mainType*100// 主力户型 + d['AK'] = detail.mainTypeArea*100// 主力户型面积 + d['AL'] = detail.developer*100// 开发商 + d['AM'] = detail.estateCompany*100// 物业公司 + d['AN'] = dbo.price_coverage*100// 小区当前均价 + } + } + }else{ + def fields = DISTRICT_FIELDS_MAP.keySet().asList() + def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields) + def detail = districtDetail(regionDistricts.size(), regionDistricts); + if(detail){ + d['E'] = regionDistricts.size()//数据量 + d['F'] = detail.township*100// 乡镇街道 + d['G'] = detail.address*100// 地址 + d['H'] = detail.addrAlias*100// 地址别名 + d['I'] = detail.name*100// 名称 + d['J'] = detail.alias*100// 别名 + d['K'] = detail.alias*100// 映射别名 + d['L'] = detail.lat2*100// 经纬度(百度) + d['M'] = detail.lat*100// 经纬度(高德) + d['N'] = detail.estateType2*100// 物业类型 + d['O'] = detail.buildType*100// 建筑类型 + d['P'] = detail.streetCommunity*100// 街道社区 + d['Q'] = detail.committee*100// 居委会 + d['R'] = detail.businessQuarter*100// 商圈 + d['S'] = detail.schools*100// 对口小学 + d['T'] = detail.kindergartens*100// 对口幼儿园 + d['U'] = detail.loudongCount*100// 楼栋总数 + d['V'] = detail.householdCount*100// 户数 + d['W'] = detail.estateCharge*100// 物业费 + d['X'] = detail.carSiteCount*100// 停车位 + d['Y'] = ""// 电梯 + d['Z'] = detail.propertyLife*100// 产权年限 + d['AA'] = detail.houseProperty*100// 产权性质 + d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分) + d['AC'] = detail.landProperty*100 // 小区性质 + d['AD'] = detail.finishBuildingDate*100// 竣工时间 + d['AE'] = detail.openDate*100// 交房时间 + d['AF'] = detail.allArea*100// 占地面积 + d['AG'] = detail.buildArea*100// 建筑面积 + d['AH'] = detail.cubagePercent*100// 容积率 + d['AI'] = detail.virescencePercent*100// 绿化率 + d['AJ'] = detail.mainType*100// 主力户型 + d['AK'] = detail.mainTypeArea*100// 主力户型面积 + d['AL'] = detail.developer*100// 开发商 + d['AM'] = detail.estateCompany*100// 物业公司 +// d['AN'] = ""// 小区当前均价 + } + } + }) + } + + + + def districtDetail(def disCount, def regionDistricts) { + + def countMap = [:] + for(def dbo : regionDistricts) { + DISTRICT_FIELDS_MAP.each() {key,skey-> + if (isValid(dbo.get(key))) { + if(countMap.containsKey(key)) { + countMap.put(key, countMap.get(key) + 1) + }else { + countMap.put(key, 1) + } + } + } + } + + def districtDetail = [:]; + DISTRICT_FIELDS_MAP.each() {key,skey-> + if(countMap.containsKey(key)) { + districtDetail.put(skey, roundDouble(countMap.get(key), disCount)) + }else { + districtDetail.put(skey, 0) + } + } + + + if(countMap.containsKey("lat") && countMap.containsKey("lng")) { + districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount)); + }else { + districtDetail.put('latLng', 0); + } + + if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) { + districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount)); + }else { + districtDetail.put('latLng2', 0); + } + return districtDetail; + } + + + def CONTAINS_KWS = ["...", "暂无", "待定", "未知"] + def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."] + def isValid(def value) { + if (!value) { + return false; + } + + if (value instanceof String) { + value = value.trim() + for(String ck : CONTAINS_KWS) { + if(value.contains(ck)) { + return false; + } + } + for(String ek : EQUALS_KWS) { + if(ek.equals(value)) { + return false; + } + } + } else if (value instanceof Number) { + if (value <= 0) { + return false; + } + } else if (value instanceof Collection || value instanceof Map) { + if (value.size() == 0) { + return false; + } + } + return true; + } + + + double roundDouble(int a, int base) { + if(a == 0 || base == 0) { + return 0 + } + return ((double)a / (double)base * 10000.0).toInteger() / 10000.0 + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index a04bae008..9ef0881e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = ["漳州"]//CommonDatautils.findAllCity().reverse() + def citys = CommonDatautils.findAllCity().reverse() def analysRets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 70249a1f9..db058c5a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -95,7 +95,7 @@ class FangjiaClientUtils { try { if(address){ address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy new file mode 100644 index 000000000..9ec05ed28 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 写字楼字段清洗入库(自动入库) + */ +@BatchOwner(Watcher.GaoJuan) +class OfficeFieldCleanBatch extends BaseBatch{ + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + static main(args) { + runBatch(new OfficeFieldCleanBatch(), args) + } + + @Override + void process(Object args) { + def citys = CommonDatautils.findAllCity().reverse() + for(def c : citys){ + def offices = findAllOffice(c) + strandField(offices) + } + + } + + + def strandField(offices){ + def fields = ["property_fee","build_area","","","","","","","","","",""] + for(def office : offices){ + fields.each{ + def newVal = null + if(it == "property_fee"){ + newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼") + newVal = newVal.toString() + } + + if(it =="all_area" || it =="build_area"){ + newVal = FiledStrand.processArea(office.get(it)) + if(newVal && newVal > 5000){ + newVal = newVal.toString() + } + } + + if(it =="name"){ + newVal = FiledStrand.processName(office.get(it)) + if(newVal){ + newVal = newVal.toString() + } + } + } + } + + } + + + def findAllOffice(city){ + def sql = "select * from office where city='${city}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + rets << d + }) + return rets + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index a59b0747a..c26ded24c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -144,5 +144,8 @@ class SeaweedDataMatchProcess { } + public static void main(def agrs){ + doMatch("上海", "静安", "金家巷西弄", null) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy index aa3c3c62c..6b0f2112c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy @@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250& stat() def stat(){ - def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1) + def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1) def map=[:] for(def d : datas){ def count = map.get(d['C']) @@ -69,8 +69,8 @@ def findLogs(){ def findFangjiaLogs(){ - def startDate = DateUtil.parseDate("20230329", "yyyyMMdd") - def endDate = DateUtil.parseDate("20230429", "yyyyMMdd") + def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") + def endDate = DateUtil.parseDate("20230531", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def list = [] @@ -84,14 +84,21 @@ def findFangjiaLogs(){ QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false); - def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") +// def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") +// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) + + def q = QueryBuilders.fuzzyQuery("request", "/enquire") QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); println houses.size() if (houses) { for (def d : houses) { def reqStr = d.request.toString() + if(!reqStr.contains("/enquire")){ + continue + } def mid = null if (reqStr.contains("mid=")) { @@ -107,5 +114,5 @@ def findFangjiaLogs(){ startDate = nextDate } - ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志") + ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志") } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy new file mode 100644 index 000000000..626af0640 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy @@ -0,0 +1,232 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang.StringUtils +import org.apache.lucene.queryparser.classic.QueryParser + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class HouseAnaly { + + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new HouseAnaly() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def allRets = [] + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"]) + def rets = processing(districts) + println "[${city}] 写文件 "+rets.size() + allRets.addAll(rets) + } + ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析") + } + + def processing(dists){ + def rets = new CopyOnWriteArrayList() + rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i=1){ + def ret = analy(dist) + if(ret){ + rets.addAll(ret) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + return rets; + } + + def analy(dist){ + def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是"); + def alias = dist.alias + for(def aliasName : alias){ + if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){ + def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否"); + if(aliaRet){ + masterRet.addAll(aliaRet) + } + } + } + if(masterRet.size()>=2){ + def masterAvgPrice = 0 + def masterMidPrice = 0 + def back = false + for(def row : masterRet){ + if(row.get(4)=="是"){ + masterAvgPrice = row.get(8) + masterMidPrice = row.get(9) + }else{ + def pre1 = 0 + def pre2 = 0 + if(masterAvgPrice>0){ + pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice) + } + if(masterMidPrice>0){ + pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice) + } + if(pre1>=20 || pre2>=20){ + back = true + } + } + } + if(back){ + return masterRet + } + + } + return null; + } + + //小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量 + def runAnalying(id, city, region, name, flag){ + def houses = findHouse(city, region, name); + def rets = [] + if(houses && houses.size()>=5){ + def prices = [] + for(def house : houses){ + prices << house.avg_price + } + prices = prices.sort() + def average = (int)(prices.sum() / prices.size()) + def midPrice = getMidPrice(prices) + def price10 = null + def price90 = null + if(houses.size()>=10){ + def postion = Math.round(houses.size()/10) + price10 = prices.get((int)(postion-1)) + price90 = prices.get((int)(houses.size()-postion-1)) + } + def countMap = classType(houses) + +// def pre = java.lang.Math.abs((midPrice-average)*100/average) +// if(pre>=20){ + rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")] +// } + } + return rets + } + + def classType(houses){ + def district = 0 + def villa = 0 + def business = 0 + for(def h : houses){ + if(h.estate_type){ + if(h.estate_type == "别墅"){ + villa = villa + 1 + continue + }else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){ + business = business + 1 + continue + } + } + def valla = isVilla(h.area, h.name, h.height, h.room) + if(valla){ + villa = villa + 1 + continue + } + district = district + 1 + } + return ["district":district, "villa":villa, "business":business] + } + + + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + def getMidPrice(def prices){ + def mid = prices.size()/2 + if(mid > (int)mid){ + return prices.get((int)mid) + }else{ + def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1)) + return (int)(total/2) + } + return null + } + + + def findHouse(def city, def region, def name){ + def houses = [] + try{ + name = escapeExprSpecialWord(name) + String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'" + ElasticUtils.executeSqlQuery(sql,{ d-> + def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type] + houses << r +// println r + }) + }catch(Exception e){ + e.printStackTrace() + } + + return houses + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy index ee95b0621..adc26463e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy @@ -21,13 +21,23 @@ class GaoDeMapSearchTest { static main(args) { def task = new GaoDeMapSearchTest(); - def path = "D:\\数据\\光大\\光大测试-V2.xlsx" + def path = "D:\\数据\\富民\\富民测试500.xlsx" // task.structureV2(path) // task.match(path) // task.mapSearch(path) - task.baiduMapSearch(path) - task.checkTrue(path) +// task.baiduMapSearch(path) +// task.checkTrue(path) + task.addFinancialLevel(path) + } + + def addFinancialLevel(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A']) + if(results){ + row['V'] = results.consumptionRating + } + }); } def checkTrue(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 4961abd56..1068d1f84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + //广州,佛山 重庆 static main(args) { def task = new BeiKeCrawlByChrome(); @@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome { println city +" 已运行" continue } + def newCity = RegionData.formatCity(city) + if(!newCity){ + if(city=="昆山"){ + newCity = "苏州" + } + println city +" 无法格式化" + } def url = m[1] def newUrl = url+"/chengjiao/" driver.get(newUrl) @@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome { def href = url+ r.attr("href") println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, city, href, regionText) + if(city == "昆山"){ + regionText = "昆山" + } + + def detailUrls = getPageDeail(driver, newCity, href, regionText) println url +" 详情链接:"+detailUrls.size() } } @@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e, region) + def dbo = findTrade(durl , "贝壳找房") + if (!dbo) { + parse(city, e, region, driver) + } }else{ stop = true; break; @@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e, region){ + def parse(city,e, region, driver){ try { + def detailUrl = e.select(".title a").attr("href") + + driver.get(detailUrl) + def detailDoc = checkDocument(driver, detailUrl); + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() def arrs = title.split(" ") + if(title.contains("车位")){ + detail.type = "车位" + } detail.city = city detail.district_name = arrs[0] detail.region = region//findRegionByThirdPary(detail.city, detail.district_name) @@ -266,28 +289,42 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" + def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + if(liftStr){ + detail.lift = DataFormatUtils.parseLift(liftStr) + detail.lift_household = DataFormatUtils.parseHousehold(liftStr) + } + detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡","")) if(detail.total_price && detail.avg_price){ - def detailUrl = e.select(".title a").attr("href") - detail.url = detailUrl - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + detail.url = detailUrl + + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + + detail.floor_info = floorStr + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorInfo && floorInfo.isNumber()) { + detail.height = floorInfo.toInteger() + } } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } }catch(Exception ex){ @@ -392,16 +429,16 @@ public class BeiKeCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def url, source){ def q = new BasicDBObject() // q.put("city", detail.city) // q.put("region", detail.region) // q.put("district_name", detail.district_name) // q.put("area", detail.area) // q.put("total_price", detail.total_price) - q.put("url", detail.url) + q.put("url", url) // q.put("trade_date", detail.trade_date) - q.put("source", detail.source) + q.put("source", source) def dbo = DEAL_PRICE_COLL.findOne(q) return dbo } @@ -446,12 +483,15 @@ public class BeiKeCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; - Thread.sleep(10000) + Thread.sleep(20000) }catch(Exception e){ - Thread.sleep(10000) + Thread.sleep(15000) } return driver } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy new file mode 100644 index 000000000..530579eb8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.paimai + +class DataFormatUtils { + + static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000] + + static def parseLift(def str){ + if(str && str.contains("梯")){ + def lift = str.toString().substring(0, str.toString().indexOf("梯")) + if(lift){ + def val = chineseIntegerPart2Digit(lift) + if(val){ + return (int)val + } + } + } + return null + } + + static def parseHousehold(def str){ + if(str && str.contains("梯") && str.contains("户")){ + def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户")) + def val = chineseIntegerPart2Digit(household) + if(val){ + return (int)val + } + } + return null + } + + + static def chineseIntegerPart2Digit(String chineseDigit) + { + Double finalDigit = 0.0; + Integer r = 1; + + for(int i = chineseDigit.length()-1; i>=0;i--) + { + Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i))); + + if( val >= 10 && i == 0 ) + { + if (val > r) + { + r = val; + finalDigit = finalDigit +val; + } + else + { + r = r * val; + } + } + else if(val >= 10) + { + if (val > r) + { + r = val; + } + else + { + r = r * val; + } + } + else + { + finalDigit = finalDigit + r*val; + } + } + return finalDigit; + } + + static String getValue(String s) { + if (s && s.contains("暂无")) { + return null + } + return s + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index aaf0c9c0d..0c82ca47e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -196,9 +196,22 @@ public class LianjiaCrawlByChrome { println city +" 已运行" continue } + def newCity = RegionData.formatCity(city) + if(!newCity){ + if(city=="昆山"){ + newCity = "苏州" + } + println city +" 无法格式化" + } def cityUrl = ret[1] def newUrl = cityUrl+"chengjiao/" - driver.get(newUrl) + try{ + driver.get(newUrl) + }catch(Exception e){ + println newUrl +" 链接有问题!" + continue + } + //def detailUrls = getPageUrls(driver, cityUrl) Thread.sleep(10000) def doc = checkDocument(driver, newUrl); @@ -208,9 +221,11 @@ public class LianjiaCrawlByChrome { for(def r : regions){ def regionText = r.text() def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href") - + if(city == "昆山"){ + regionText = "昆山" + } println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, city, href, regionText) + def detailUrls = getPageDeail(driver, newCity, href, regionText) println cityUrl +" 详情链接:"+detailUrls.size() } } @@ -284,7 +299,10 @@ public class LianjiaCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e, regionText) + def dbo = findTrade(durl , "链家在线") + if (!dbo) { + parse(city, e, regionText, driver) + } }else{ stop = true; break; @@ -302,85 +320,106 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e, regionText){ + def parse(city,e, regionText, driver){ try { - def title = e.select(".title a").text() - if (title) { - def detail = new BasicDBObject() - def arrs = title.split(" ") - detail.city = city - detail.district_name = arrs[0] - if(regionText){ - detail.region = regionText - }else{ - detail.region = findRegionByThirdPary(detail.city, detail.district_name) - } - if(detail.region){ - detail.region = RegionData.formatRegion(city, detail.region) - } - - def roomStr = arrs[1] - if (roomStr) { - if (roomStr.indexOf("室") > 0) { - def room = roomStr.substring(0, roomStr.indexOf("室")) - if(room && room.isNumber()){ - detail.room = room.toInteger() - } + def price = e.select(".totalPrice .number").text() + if(price){ + def detailUrl = e.select(".title a").attr("href") + driver.get(detailUrl) + def detailDoc = checkDocument(driver, detailUrl); + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + if(title.contains("车位")){ + detail.type = "车位" } - if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) - if(hall && hall.isNumber()){ - detail.hall = hall.toInteger() - } + detail.city = city + detail.district_name = arrs[0] + if(regionText){ + detail.region = regionText + }else{ + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + } + if(detail.region){ + detail.region = RegionData.formatRegion(city, detail.region) } - } - if("车位"!=roomStr){ - if(arrs && arrs.size()>=3){ - def areaStr = arrs[2].replace("平米", "") - if (areaStr) { - detail.area = areaStr.toDouble() + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + def room = roomStr.substring(0, roomStr.indexOf("室")) + if(room && room.isNumber()){ + detail.room = room.toInteger() + } + + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hall && hall.isNumber()){ + detail.hall = hall.toInteger() + } } } - def price = e.select(".totalPrice .number").text() - if (price && price.isNumber()) { - detail.total_price = (int)(price.toDouble() * 10000) - } + if("车位"!=roomStr){ + if(arrs && arrs.size()>=3){ + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + } - if (detail.total_price && detail.area) { - detail.avg_price = (int) (detail.total_price / detail.area) - } - def tradeStr = e.select(".dealDate").text() - if (tradeStr) { - detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") - } - detail.source = "链家在线" - detail.url = e.select(".title a").attr("href") - if(detail.total_price && detail.avg_price){ - def dbo = findTrade(detail) - if (!dbo) { + if (price && price.isNumber()) { + detail.total_price = (int)(price.toDouble() * 10000) + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + detail.url = detailUrl + + def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + if(liftStr){ + detail.lift = DataFormatUtils.parseLift(liftStr) + detail.lift_household = DataFormatUtils.parseHousehold(liftStr) + } + detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层","")) + + if(detail.total_price && detail.avg_price){ def houseInfo = e.select(".houseInfo").text() def houseInfos = houseInfo.split("\\|") detail.direction = houseInfos[0] detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() + + detail.floor_info = floorStr if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() + def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr2 && floorStr2.isNumber()) { + detail.height = floorStr2.toInteger() } } - detail.create_date = new Date() detail.m_date = new Date() println detail DEAL_PRICE_COLL.save(detail) } } - } } }catch(Exception ex){ @@ -484,16 +523,16 @@ public class LianjiaCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def url, source){ def q = new BasicDBObject() // q.put("city", detail.city) // q.put("region", detail.region) // q.put("district_name", detail.district_name) // q.put("area", detail.area) // q.put("total_price", detail.total_price) + q.put("url", url) // q.put("trade_date", detail.trade_date) - q.put("url", detail.url) - q.put("source", detail.source) + q.put("source", source) def dbo = DEAL_PRICE_COLL.findOne(q) return dbo } @@ -535,7 +574,7 @@ public class LianjiaCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; - Thread.sleep(10000) + Thread.sleep(15000) }catch(Exception e){ Thread.sleep(10000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy new file mode 100644 index 000000000..949e55f51 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.gj.region + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 格式化其它网站区域 + */ + + +def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + +DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl) + + + +def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1) +for(def d : datas){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("recognizeCity", d['F']) + q.put("recognizeRegion", d['A']) + def count = SERVER_AREA_COLL.count(q) + if(!count){ + def obj = new BasicDBObject() + obj.put("city", d['A']) + obj.put("recognizeRegion", d['A']) + obj.put("recognizeCity", d['F']) + obj.put("createdDate", new Date()) + SERVER_AREA_COLL.save(obj) + }else{ + println d['A']+" "+ d['F'] +" 已存在" + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 96709e230..9c6952a70 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -237,7 +237,7 @@ public class DbUtils{ def lng = district.lng / 10000000000.0 sdq.put("sd.point", lat + "," + lng); sdq.put("sd.sfield", "lat_lng"); - sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0)); + sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0)); query.append('$sd', sdq) def keys = new BasicDBObject() From 3f860273a0b2e13870fa62ed7821688189fb3a6a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 9 Jun 2023 15:30:45 +0800 Subject: [PATCH 46/50] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 5a38c237e..ed89f59ac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -63,7 +63,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); cityRun(citys, startDate, endDate, cityCodeMap) regionRun(citys, startDate, endDate, cityCodeMap) - districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) +// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } def cityRun(citys, startDate, endDate, cityCodeDataMap){ From 0b84d882c79225fbb422a3238db4eeeb4ef3ba2b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 9 Jun 2023 18:16:08 +0800 Subject: [PATCH 47/50] =?UTF-8?q?=E6=B1=9F=E5=8D=97=E8=A1=8C=E9=80=89?= =?UTF-8?q?=E6=88=BF=E6=BA=90=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datastore/CzbankSampleService.java | 6 +- .../datastore/CzbankSampleServiceImpl.java | 220 +++++++++++++++++- 2 files changed, 222 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleService.java index 53906f2e6..26e59f609 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleService.java @@ -1,10 +1,13 @@ package com.fangjia.service.datastore; +import com.fangjia.dal.mongo.model.DistrictStat; import com.fangjia.service.bo.Collateral; +import com.fangjia.service.bo.SampleWrapper; import com.fangjia.service.bo.ShopSample; import com.fangjia.service.datastore.bean.CollateralSampleQueryBean; import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean; +import java.util.Date; import java.util.List; /** @@ -27,4 +30,5 @@ public interface CzbankSampleService { */ List findShopSample(ShopCollateralSampleQueryBean queryBean, int max); -} + + } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java index 52cea9435..3a0730280 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java @@ -46,7 +46,7 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ "totalPrice", "avgPrice", "area", "roomNumber", "fitment", "direction", "floor", "height", "tradeDate", "address"}; private static String[] HOUSE_FIELDS = new String[]{"city", "region", "districtName", "totalPrice", "avgPrice", - "area", "fitment", "direction", "floor", "height", "publishDate", "address", "estateType"}; + "area", "fitment", "direction", "floor", "height", "publishDate", "address", "estateType","url"}; public static String[] TAGS =new String[]{"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"}; @@ -187,7 +187,9 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ districtStat = districtStatService.findOneDistrictStatsSort(district.getCity(), district.getRegion(), district.getName(), DateUtil.add( new Date(), -7)); } List querySamples= new ArrayList(); - List samples = findSampleByDistrict(district.getCity(), district.getRegion(), district.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); +// List samples = findSampleByDistrict(district.getCity(), district.getRegion(), district.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); + List samples = findJnbankSampleByDistrict(district.getCity(), district.getRegion(), district.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); + // scoreForSample(samples, target); sortForSample(samples, target); for(SampleWrapper sample : samples){ @@ -199,7 +201,9 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ List districts = districtService.findNearestDistrict(district, 1000, 5); for(District d : districts){ d = districtService.findById(d.getId()); - List nearlySample = findSampleByDistrict(d.getCity(), d.getRegion(), d.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); + //List nearlySample = findSampleByDistrict(d.getCity(), d.getRegion(), d.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); + List nearlySample = findJnbankSampleByDistrict(d.getCity(), d.getRegion(), d.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); + for(SampleWrapper sample : nearlySample){ sample.setProperty(d); } @@ -499,6 +503,27 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ }); List prices = new ArrayList(); +// for(SampleWrapper sample : samples){ +// if(!prices.contains(sample.getUnitPrice())){ +// double min = targetSample.getStrctureArea() - 20; +// double max = targetSample.getStrctureArea() + 20; +// if(sample.getStrctureArea()>=min && sample.getStrctureArea()<=max){ +// prices.add(sample.getUnitPrice()); +// } +// } +// } +// if(prices.size()<10){ +// prices.clear(); +// for(SampleWrapper sample : samples){ +// if(!prices.contains(sample.getUnitPrice())){ +// prices.add(sample.getUnitPrice()); +// if(prices.size()>=10){ +// break; +// } +// } +// } +// } + for(SampleWrapper sample : samples){ if(!prices.contains(sample.getUnitPrice())){ prices.add(sample.getUnitPrice()); @@ -708,6 +733,195 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ } + /** + * 江南行查找小区样本(优先使用贝壳链家) + * @param city + * @param region + * @param name + * @param alias + * @param estateType 用户查询物业类型 + * @param startDate + * @param distEstateType 小区物业类型 + * @return + */ + private List findJnbankSampleByDistrict(String city, String region, String name, List alias, String masterName, String distEstateType, String estateType, Date startDate, DistrictStat districtStat){ + + List names = new ArrayList(); + names.add(ClientUtils.escapeQueryChars(name)); + if(alias != null){ + for(String alia : alias){ + if(StringUtils.isNotBlank(alia) && alia.length() >= 2 && !names.contains(alia)){ + names.add(ClientUtils.escapeQueryChars(alia)); + } + } + } + if(StringUtils.isNotBlank(masterName)){ + names.add(ClientUtils.escapeQueryChars(masterName)); + } + + List samples = new ArrayList(); + List nolianjiaSamples = new ArrayList(); + + List houseQuery = new ArrayList(); + houseQuery.add(new QueryParam("city", city )); + houseQuery.add(new QueryParam("region", region )); + if(districtStat != null && districtStat.getAvgPrice() > 0){ + int avgPrice = districtStat.getAvgPrice(); + houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, (int)(avgPrice * 0.5 )));//0.85 + houseQuery.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_EQUAL_OPERATOR, (int)(avgPrice * 1.5 )));//1.15 + } + BooleanQuery nameQuery = new BooleanQuery(); + int c = 0; + for (String n : names) { + if (StringUtils.isNotBlank(name)) { + if (name.indexOf("(") > -1 || name.indexOf(")") > -1 || name.indexOf("(") > -1 || name.indexOf(")") > -1 + | name.indexOf("[") > -1 || name.indexOf("]") > -1 || name.indexOf("-") > -1 || name.indexOf("{") > -1 + || name.indexOf("}") > -1 || name.indexOf("{") > -1 || name.indexOf("}") > -1 || name.indexOf(":") > -1|| name.indexOf("\\") > -1) { + continue; + } + c++; + nameQuery.add(new TermQuery(new Term("district_name", n)), BooleanClause.Occur.SHOULD); + } + } + + if(nameQuery.clauses().size()==0){ + return samples; + } + BooleanQuery srQuery = new BooleanQuery(); + srQuery.add(nameQuery, BooleanClause.Occur.MUST); + houseQuery.add(new QueryParam("$sr", new SrQuery(srQuery))); + if(startDate != null){ + houseQuery.add(new QueryParam("publishDate", QueryParam.GREATER_THAN_OPERATOR, startDate)); + } + + List houseSortParams = new ArrayList(); + houseSortParams.add(new SortParam("publishDate", false)); + List houses = houseDao.findAllByParams(houseQuery, houseSortParams, 0, 500, HOUSE_FIELDS); + int height = 0; + int maxFloor = 0; + boolean allSampleHasHeight = true; + for(House house : houses){ + if(StringUtils.isNotBlank(estateType) && estateType.equals("01")){//01:普通住宅 02:别墅 + if(StringUtils.isNotBlank(house.getEstateType()) && house.getEstateType().contains("别墅")){ + continue; + } + if(isVilla(house.getArea(), house.getName(), house.getHeight(), house.getRoom())){ + continue; + } + }else if(StringUtils.isNotBlank(estateType) && estateType.equals("02")){ + if(house.getHeight()!=null && house.getHeight()>=10){ + continue; + } + if(house.getFloor()!=null && house.getFloor()>=10){ + continue; + } + } + + SampleWrapper sample = convertHouse2Sample(house); + if(isAvailableSample(sample)){ + if(house.getUrl().contains("lianjia.com") || house.getUrl().contains("ke.com")){ + samples.add(sample); + }else{ + nolianjiaSamples.add(sample); + } + } + if(sample.getAboveFloorAmount() == null){ + allSampleHasHeight = false; + }else if(sample.getAboveFloorAmount() > 0 && sample.getAboveFloorAmount() > height && sample.getAboveFloorAmount() < 100){ + height = sample.getAboveFloorAmount() ; + } + if(sample.getFloorNumber() != null && sample.getFloorNumber() > maxFloor){ + maxFloor = sample.getFloorNumber(); + } + //浙商测试特意添加 + if(house.getDistrictName().equals("新泾三村") && house.getArea()>=35 && house.getArea()<=55){ + sample.setDoorNo("502"); + sample.setBuildingNoName("56"); + } + + if(StringUtils.isNotBlank(house.getEstateType())){ + if(house.getEstateType().contains("商业") || house.getEstateType().contains("商住") || house.getEstateType().contains("办公")){ + sample.setBuildingName(house.getDistrictName() +"(商住)"); + } + }else{ + if(StringUtils.isNotBlank(distEstateType) && (distEstateType.contains("商住") && (!name.contains("商住") || !name.contains("商业")) )){ + sample.setBuildingName(house.getDistrictName() + "(商住)"); + } + } + + } + List dealPriceQuery = new ArrayList(); + dealPriceQuery.add(new QueryParam("city", city )); + dealPriceQuery.add(new QueryParam("region", region )); + dealPriceQuery.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names)); + if(startDate != null){ + dealPriceQuery.add(new QueryParam("tradeDate", QueryParam.GREATER_THAN_OPERATOR, startDate )); + }else{ + dealPriceQuery.add(new QueryParam("tradeDate", QueryParam.GREATER_THAN_OPERATOR, DateUtil.add(new Date(), Calendar.MONTH, -3) )); + } + List dealPriceSortParams = new ArrayList(); + dealPriceSortParams.add(new SortParam("tradeDate", false)); + List dealPrices = dealPriceDao.findAllByParams(dealPriceQuery, dealPriceSortParams, 0, 100, DEAL_PRICE_FIELDS); + for(DealPrice dealPrice : dealPrices){ + SampleWrapper sample = convertDealPrice2Sample(dealPrice); + if(isAvailableSample(sample)){ + samples.add(sample); + } + if(sample.getAboveFloorAmount() == null){ + allSampleHasHeight = false; + }else if(sample.getAboveFloorAmount() > 0 && sample.getAboveFloorAmount() > height && sample.getAboveFloorAmount() < 100){ + height = sample.getAboveFloorAmount() ; + } + if(sample.getFloorNumber() != null && sample.getFloorNumber() > maxFloor){ + maxFloor = sample.getFloorNumber(); + } + } + if(!allSampleHasHeight){ + if(height == 0){//如果没有楼高到楼栋库里寻找最高的楼高 + height = findHeight(city, region, name); + } + if(height == 0){//如果楼栋库没有使用房源中的最大楼层 + height = maxFloor; + } + if(height == 0){//没有楼高信息使用默认的楼高 + if(estateType == "01"){//住宅默认6楼 + height = 6; + }else{//别墅默认三村 + height = 3; + } + } + } + + FangjiaOfficialAreaMap fangjiaOfficialAreaMap = fangjiaOfficialAreaService.convertFangjiaToOfficial(city, region); + + if(samples.size()<10){ + samples.addAll(nolianjiaSamples); + } + + for(SampleWrapper sample : samples){ + if(sample.getAboveFloorAmount() == null){ + sample.setAboveFloorAmount(height); + } + if(sample.getFloorNumber() == null){ + if("01".equals(estateType)){ + sample.setFloorNumber((sample.getAboveFloorAmount()/2)); + }else{ + sample.setFloorNumber(1); + } + } + if(fangjiaOfficialAreaMap != null){ + sample.setProvince(fangjiaOfficialAreaMap.getProvinceCode()); + sample.setCity(fangjiaOfficialAreaMap.getCityCode()); + sample.setDistrict(fangjiaOfficialAreaMap.getDistrictCode()); + } + } + + + return samples; + } + + + /** * 根据小区名在楼栋库中查询最大的楼高 * @param city From 5d057a5f853aa209ae9c0dc483133ecbce7dd53b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 12 Jun 2023 13:35:40 +0800 Subject: [PATCH 48/50] =?UTF-8?q?=E5=B8=B8=E5=B7=9E=E5=8E=BB=E9=99=A4?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=B0=8F=E5=8C=BA=E5=8E=BB=E6=8E=89=E7=9A=84?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index ced116b61..430aa56e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -59,7 +59,7 @@ class CZDataBatch extends BaseBatch{ def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); districtRun(districts, statMap, commids); - FzDataUtils.removeNoUpdate(city, districts) +// FzDataUtils.removeNoUpdate(city, districts) def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = mergeShop(shops); From 4022bff4b5178498cbabd2b3d8addb6d70e351f0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 12 Jun 2023 13:40:19 +0800 Subject: [PATCH 49/50] update --- .../groovy/com/fangjia/script/commData/CommonDatautils.groovy | 2 +- .../groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 6c633d4f7..767b0b48a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -232,7 +232,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); - q.put("name", "长滩美墅三期") +// q.put("name", "长滩美墅三期") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 430aa56e2..4e23f59e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -58,8 +58,9 @@ class CZDataBatch extends BaseBatch{ def commids = findAllJnBankDatas(city) def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); + FzDataUtils.removeNoUpdate(city, districts) districtRun(districts, statMap, commids); -// FzDataUtils.removeNoUpdate(city, districts) + def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = mergeShop(shops); From ca54df8e649c45b3b00deaddfa1c07a29edfdf11 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 13 Jun 2023 13:17:44 +0800 Subject: [PATCH 50/50] =?UTF-8?q?=E6=9D=BF=E5=9D=97=E7=BC=96=E8=BE=91?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/webapp/district/edit.ftl | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl index 9c9421494..482d0260b 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl @@ -245,19 +245,33 @@ (浣忓畢銆佸叕瀵撱�佺嫭鏍嬪埆澧呫�佽仈鎺掑埆澧呫�佸弻鎷煎埆澧�...) --> - +<#-- --> +<#-- --> +<#-- <#if markMap?? && markMap["block"]?? && markMap["block"]?is_number && markMap["block"] == 1>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- 鏉垮潡--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> + + - <#if markMap?? && markMap["block"]?? && markMap["block"]?is_number && markMap["block"] == 1> - - <#else> - - - 鏉垮潡 + <#if markMap?? && markMap["block"]?? && markMap["block"]?is_number && markMap["block"] == 1> + + <#else> + + + 鏉垮潡 - + + <#if markMap?? && markMap["loop_line_info"]?? && markMap["loop_line_info"]?is_number && markMap["loop_line_info"] == 1>