From 7dff8154c5a449b5f73f80fed61dfb6e93b60c96 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 17 Jul 2021 20:54:44 +0800 Subject: [PATCH 001/111] add ccb batch file --- .../BigdataCommunityBatch.groovy | 154 ++++++ .../ccbevaluation/BigdataDistrictBatch.groovy | 91 +++ .../ccbevaluation/BigdataForsaleBatch.groovy | 520 +++++++++++++++++ .../ccbevaluation/BigdataForsaleBatch2.groovy | 521 ++++++++++++++++++ .../ccbevaluation/CcbCommonUtils.groovy | 61 ++ .../script/ccbevaluation/CityBatch.groovy | 100 ++++ .../ccbevaluation/EvaluateRateBatch.groovy | 64 +++ .../ccbevaluation/GuiDingPriceBatch.groovy | 99 ++++ .../ccbevaluation/StagingDistrictBatch.groovy | 85 +++ .../ccbevaluation/ThirdPartyPriceBatch.groovy | 177 ++++++ .../gj/data/BestDistrictPriceExport.groovy | 174 ++++++ .../gj/district/SourceDistDataMatch.groovy | 265 +++++++++ .../script/gj/region/MapAreaUpdate.groovy | 65 +++ 13 files changed, 2376 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy new file mode 100644 index 000000000..3913caee7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -0,0 +1,154 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 建行模型估值项目 + * 小区数据同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class BigdataCommunityBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new BigdataCommunityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + + def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + "总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + "乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] + + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) + def rets = createCityDatas(cityCodeDatas, distCodeMap, dists) + rows.addAll(rets) + println city +" "+rets.size() + + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); + } + + def allDatas = [] + allDatas << headers + allDatas.addAll(rows) + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false); + println "txt 生成" + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") + } + + def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){ + def rets = [] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + def data = [] + data << m.dist_code + data << m.dist_name + data << m.dist_spell + data << m.city_id + data << m.dist_code + data << index + data << "" + data << "" + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + data << "" + data << "" + data << "" + data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + index++ + allRows << data + } + return allRows; + } + + def findDistrictCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + q.put("district", new BasicDBObject('$ne',"市辖区")) + def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + def lists = [] + for(def db : dbcursor){ + if(db.district_code){ + def m = [:] + m.id = db._id.toString() + m.dist_name = db.fangjia_district + m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") + m.city_id = db.city_code + m.dist_code = db.district_code + lists << m + } + } + return lists; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy new file mode 100644 index 000000000..7e97743c6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -0,0 +1,520 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataForsaleBatch extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + + static main(args) { + runBatch(new BigdataForsaleBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731); + } + } + + def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){ + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println city +" 小区量:"+districts.size() + + def retMap = new ConcurrentHashMap() + for(def i=0;i0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = city + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(rootMap.get(seawater.root)){ + rowMap.root = rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + } + } + return rets; + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + /* + * 生成一个用不重复的ID + */ + def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) + } + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy new file mode 100644 index 000000000..c896476d1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy @@ -0,0 +1,521 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataForsaleBatch2 extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + + def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + + static main(args) { + runBatch(new BigdataForsaleBatch2(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def distMap = getDistrictMap(city) + exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731); + } + } + + def getDistrictMap(def city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + if(d.alias){ + d.alias.each{name-> + if(!distMap.containsKey(d.region+"##"+name)){ + distMap.put(d.region+"##"+name, d) + } + } + } + } + return distMap; + } + + def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){ + println city +" 小区key:"+distMap.size() + def cityCode = cityCodeDatas.city_code + def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap) + def allDatas = createSellDatas(city, retMap); + def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true); + println "txt文件生成 " + } + + + + def createSellDatas(def city, def map){ + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(map.size()) + println city +"总房源量:"+map.size() + + def datas = [] + map.each{k,m-> + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + datas << converData(m) + println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return datas ; + } + + def converData(def m){ + def data = [] + data << m.city_code + data << m.houseId + data << "" + data << m.city + data << m.region_code + data << m.region + data << m.district_id + data << m.district_name + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << m.address==null?"":m.address==null + data << m.build_year==null?"":m.build_year + data << m.proptype==null?"":m.proptype + data << m.propyears==null?"":m.propyears + data << m.avg_price==null?"":m.avg_price + data << m.total_price==null?"":m.total_price + data << "" + data << m.floor==null?"":m.floor + data << m.height==null?"":m.height + data << m.estate_type==null?"":m.estate_type + data << m.house_structure_code==null?"":m.house_structure_code + data << m.build_structure_code==null?"":m.build_structure_code + data << m.area==null?"":m.area + data << "" + data << m.house_type==null?"":m.house_type + data << m.room==null?"":m.room + data << m.hall==null?"":m.hall + data << "" + data << m.toilet==null?"":m.toilet + data << m.direction_code==null?"":m.direction_code + data << m.fitment_code==null?"":m.fitment_code + data << "" + data << "" + data << m.p_date==null?"":m.p_date + data << "" + data << "" + data << m.root + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << m.c_date==null?"":m.c_date + data << m.m_date==null?"":m.m_date + data << m.source + data << "" + return data + } + + + def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def retMap = new ConcurrentHashMap() + println city +" 没过滤前:"+ houses.size() + if(houses){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(houses.size()) + + for (int k=0;k60){ + return null + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + return null + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + return null + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + return null + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + return null + } + + if(seawater.avg_price && seawater.avg_price<10){ + return null + } + + } + rowMap.houseId = seawater.id + rowMap.city = city + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(rootMap.get(seawater.root)){ + rowMap.root = rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + + def regionCode = distCodeMap.get(rMap.region) + def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) + def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) + def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) + def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type); + def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); + rowMap.city_code = cityCode + rowMap.region = rMap.region + rowMap.region_code = regionCode + rowMap.district_id = rMap.get("_id").toString() + rowMap.district_name = rMap.name + rowMap.build_year = buildYear; + rowMap.proptype = housePro + rowMap.propyears = propertyYear; + rowMap.estate_type = estateType + rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall); + rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction) + rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment); + rowMap.address = rMap.address + rowMap.build_structure_code = struCode + rowMap.house_structure_code = null//getHouseStructureCode(); + rowMap.source = "FJW" + return rowMap + + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + /* + * 生成一个用不重复的ID + */ + def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) + } + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy new file mode 100644 index 000000000..37c99bc96 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CcbCommonUtils { + + public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] +// public static def citys = ["北京"] + + public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] + + public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") + + public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") + + public static def findCityCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count >1){ + def data = null + for(def db : dbcursor){ + if(db.city_code){ + data = db + break + } + } + return data; + } + } + + public static def findAllDistrictCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + q.put("district", new BasicDBObject('$ne',"市辖区")) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def map = [:] + for(def db : dbcursor){ + map.put(db.fangjia_district, db.district_code) + } + return map; + } + + public static def findPersonPrice(city, date){ + def q = new BasicDBObject() + q.put("city", city) + q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date))); + def dbcursor = GUIDING_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy new file mode 100644 index 000000000..bafa9e4d8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +/** + * 建行模型估值项目 + * 城市数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class CityBatch extends BaseBatch{ + + static main(args) { + runBatch(new CityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + def index = 1 + for(def city : citys){ + def data = CcbCommonUtils.findCityCodeValue(city) + def map = [:] + map.name = city + if(data){ + map.city_code = data.city_code + map.prov_code = data.province_code + } + def cityObj = findCityInfo(city) + if(cityObj){ + map.city_spell = cityObj.spell + map.city_level = cityObj.city_grade + map.seq = index + map.ifcapital = isCapital(cityObj.province, city) + map.ifmunicipal = ifMunicipal(city) + } + index++ + rows << map + } + + def allDatas = createDatas(rows) + def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"] + + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8"); + } + + def createDatas(rows){ + def allRows = [] + rows.each { m -> + def data = [] + data << m.city_code + data << m.name + data << m.city_spell + data << m.prov_code + data << m.city_code + data << m.city_level + data << m.seq + data << CcbCodeUtils.parseFlag(m.ifcapital) + data << CcbCodeUtils.parseFlag(m.ifmunicipal) + data << "" + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "fjw" + allRows << data + } + return allRows; + } + + def findCityInfo(city){ + def q = new BasicDBObject() + q.put("display", city) + return CcbCommonUtils.SERVER_AREA_COLL.findOne(q) + } + + def isCapital(def prov, def city){ + def provCity = CcbCommonUtils.provCityMap.get(prov) + if(provCity && provCity==city){ + return "是" + } + return "否" + } + + def ifMunicipal(def city){ + if(["上海", "北京", "天津", "重庆"].contains(city)){ + return "是" + } + return "否" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy new file mode 100644 index 000000000..be208aebb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql + +/** + * 建行模型估值项目 + * 估价系数数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class EvaluateRateBatch extends BaseBatch{ + + static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver") + + static main(args) { + runBatch(new EvaluateRateBatch(), args); + } + + @Override + public void process(Object args) { + + def allDatas = createDatas(findEvaluationRate()) + def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false); + } + + def createDatas(def datas){ + def allRows = [] + datas.each { m -> + def data = [] + data << m.id.toString() + data << m.rate_type + data << m.rate + data << m.rank + data << "全国" + data << m.rank_remark + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + allRows << data + } + return allRows + } + + private def findEvaluationRate(){ + def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'"; + def rets = [] + sqlInstance.eachRow(sql,{ + rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated] + }); + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy new file mode 100644 index 000000000..234d172b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 建行模型估值项目 + * 人工指导价同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class GuiDingPriceBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new GuiDingPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + + } + + def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false); + + def rets = [] + rets << headers + rets.addAll(rows) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") + + } + + def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){ + def rets = [] + if(prices){ + for(def ps : prices){ + def dist = distMap.get(ps.region+"##"+ps.name) + if(dist){ + def ret = [] + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + if(dist.region){ + ret << distCodeMap.get(dist.region) + }else{ + ret << "" + } + ret << dist.region + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << ps.guiding_value + ret << ps.guiding_person + ret << "" + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + } + return rets; + } + + def getDistrictMap(def city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + } + return distMap; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy new file mode 100644 index 000000000..4ecac813a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +/** + * 建行模型估值项目 + * 分期数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class StagingDistrictBatch extends BaseBatch{ + + static main(args) { + runBatch(new StagingDistrictBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"]) + def fenQiMap = getFenQiMap(dists); + def rets = createCityData(dists, fenQiMap) + rows.addAll(rets) + println city +" 分期量:"+rets.size() + } + + + def results =[] + def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + results << headers + results.addAll(rows) + + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据") + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8"); + + } + + def createCityData(def dists, def fenQiMap){ + def rets = [] + for(def d : dists){ + if(d.master_district_name && d.master_district_name!="master"){ + def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name) + if(masterDist){ + def ret = [] + ret << new ObjectId().toString() + ret << d._id.toString() + ret << masterDist._id.toString() + ret << "" + ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } +// else{ +// def udata = new BasicDBObject() +// udata.put("master_district_name", null) +// udata.put("m_date", new Date()) +// println d.city +" "+d.name+" "+d._id.toString() +// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata)) +// } + } + } + return rets; + } + + def getFenQiMap(def dists){ + def fenQiMap = [:] + for(def d : dists){ + if(d.master_district_name){ + fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d) + } + } + return fenQiMap; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy new file mode 100644 index 000000000..d4e252dd5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -0,0 +1,177 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 建行模型估值项目 + * 第三方均价数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class ThirdPartyPriceBatch extends BaseBatch { + + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + + static main(args) { + runBatch(new ThirdPartyPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + def index = 0 + for(def city : citys){ + index++ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"]) + def thirdPriceMap = findThirdPartyPrice(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) + rows.addAll(rets) + println index+" "+city+" "+rets.size() + } + def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false); + + } + + + def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def rets = [] + for(def d : dists){ + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def ret = [] + ret << third._id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + if(d.region){ + ret << distCodeMap.get(d.region) + }else{ + ret << "" + } + ret << d.region + ret << third.s_date + + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + + } + return rets; + } + + + def findThirdPartyPrice(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", curDate) + def dbcursor = AVERAGE_STATISTICS.find(q) + def map = [:] + for(def db : dbcursor){ + map.put(db.region+"##"+db.name, db) + } + return map + } + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy new file mode 100644 index 000000000..af92f0489 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy @@ -0,0 +1,174 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +class BestDistrictPriceExport { + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new BestDistrictPriceExport(); + task.run() + } + + def run(){ + def city = "佛山" + def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + def rets = [] + rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"] + def distMap = findCreditHightDistrict2(city, "20210711"); + distMap.each{region, dists-> + + if(region=="三水"){ + def addIndex = 0 + for(def dist : dists){ + if(addIndex <=30){ + def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name) + def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711") + if(otherPrice){ + def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price + if(Math.abs(precent)>=0 && Math.abs(precent)<=10){ + def ret = [] + ret << "佛山" + ret << dist.region + ret << dist.district_name + ret << dist.estate_type + for(def weekend : months){ + def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend) + if(obj){ + ret << obj.avg_price + }else{ + ret << "" + } + } + ret << otherPrice.lianjia_esf_price + ret << stat.credit + rets << ret + addIndex++ + } + } + } + } + } + } + ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格") + } + + + def findDistrictPrice(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def obj = DIST_SELL_STAT.findOne(q) + return obj + } + + def findCreditHightDistrict(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def typeMap = retMaps.get(d.region) + if(!typeMap){ + typeMap = [:] + retMaps.put(d.region, typeMap) + } + + def rets = typeMap.get(d.estate_type) + if(!rets){ + rets = [] + typeMap.put(d.estate_type, rets) + } + + def count = 0 + typeMap.each {k,v-> + if(v){ + count = count + v.size() + } + } + + if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){ + def m = [:] + m.city = d.city + m.region = d.region + m.district_name = d.district_name + rets << m + }else if(rets.size() <= 20){ + def m = [:] + m.city = d.city + m.region = d.region + m.district_name = d.district_name + rets << m + } + + def total = 0 + retMaps.each {k,m-> + m.each {t,v-> + total = total + v.size() + } + } + if(total>100){ + break; + } + } + + def rows = [] + retMaps.each {k,m-> + m.each {t,v-> + rows.addAll(v) + } + } + return rows + } + + + def findCreditHightDistrict2(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + def m = [:] + m.city = d.city + m.region = d.region + m.estate_type = d.estate_type + m.district_name = d.district_name + rets << m + } +// +// def rows = [] +// retMaps.each {k,v-> +// rows.addAll(v) +// } + return retMaps; + } + + def findNearThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", 202106) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + return AVERAGE_STATISTICS.findOne(q) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy new file mode 100644 index 000000000..4687872d3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy @@ -0,0 +1,265 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.apache.lucene.queryparser.classic.QueryParser + +class SourceDistDataMatch { + + static main(args) { + def task = new SourceDistDataMatch(); +// task.run() + task.mapRun() +// task.filterName() +// task.filterType() + } + + def run(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(obj){ + d['R'] = 1 + }else{ + def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709) + if(distCount){ + d['W'] = distCount + } + } + },0) + } + + def mapRun(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N'] && !d['F']){ + def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']] + def locationMap = matchMap(dist) + if(locationMap){ + d['F'] = locationMap.source + d['G'] = locationMap.region + d['H'] = locationMap.mapName + d['I'] = locationMap.address + d['J'] = locationMap.type + d['K'] = locationMap.lng + d['L'] = locationMap.lat + d['M'] = locationMap.score + } + } + },0) + } + + def filterName(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N']) { + def name = d['C'] + if (name) { + def flag = NameProcess.filterName(name) + if (!flag) { + d['N'] = 4 + } + } + } + },0) + + } + + def filterType(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N']){ + def type = d['J'] + if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){ + + }else if(type && type.contains("商务住宅") && !type.contains("商住两用")){ + def count = 0 + if(d['S'] && d['S']!='null'){ + count = d['S'].toDouble().toInteger() + } + if(count<10){ + d['N'] = 3 + } + }else{ + if(type){ + d['N'] = 3 + } + } + } + },0) + } + + + def matchMap(def dist){ + def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") + if(!gaodeRest && dist.address){ + gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德") + } + def gdlocationMap = null + def bdlocationMap = null + if(gaodeRest){ + gdlocationMap = getMapLocation2(dist, gaodeRest); + if(gdlocationMap){ + gdlocationMap.source = "高德" + } + } + + def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + if(baiduRest){ + bdlocationMap = getMapLocation2(dist, baiduRest); + if(bdlocationMap){ + bdlocationMap.source = "百度" + } + } + + if(gdlocationMap && bdlocationMap){ + if(gdlocationMap.score>=bdlocationMap.score){ + return gdlocationMap + }else{ + return bdlocationMap + } + }else if(gdlocationMap){ + return gdlocationMap + }else if(bdlocationMap){ + return bdlocationMap + } + return null + } + + + + def findHouseCount(def city, def region, def alias, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } + + def getMapLocation2(def dist, def rets){ + def data1 = findSimilarData(dist, rets, true) + def data2 = findSimilarData(dist, rets, false) + println data1 + println data2 + if(data1 && data1.score>1){ + return data1 + }else if(data2 && data2.score>1){ + return data2 + } + return null + } + + def findSimilarData(def dist, def rets, def regionImpr){ + def lat = null + def lng = null + def region = null + def mapName = null + def address = null + def score = 0 + def type = null + def maxScore = 0 + def standardName = null + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + if(["公交车站"].contains(r.type)){ + continue + } + if(regionImpr && !r.district){ + continue + } + if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){ + continue + } + + def name = nameReplace(r.name, dist.city) + + def distName = nameReplace(dist.name, dist.city) + + def score3 = 0.0 + if(r.district && dist.region && r.district.contains(dist.region)){ + score3 = 0.2 + } + + def score1 = 0.0; + def score2 = 0.0 + if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){ + if(r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2 + } + }else{ + score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2; + if(dist.address && r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2 + } + } + + score3 = score1 + score2 + if(r.type && r.type.contains("住宅")){ + score3 = score3 + 0.2 + } + if(score3 >maxScore){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + standardName = name + address = r.address + type = r.type + score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10) + maxScore = score3 + } + println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3 + + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName] + } + return null + } + + def nameReplace(def s, city){ + if(s.contains("(") && s.contains(")")){ + def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")")) + if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){ + s = s.toString().substring(0, s.indexOf("(")) + } + } + def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim() + if(name.startsWith(city)){ + def newName = name.replace(city+"市","").replace(city,"") + if(newName && newName=="花园"){ + return name + }else{ + name = newName + } + } + return name + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy new file mode 100644 index 000000000..40748a9fd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.gj.region + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class MapAreaUpdate { + static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + static main(args) { + def task = new MapAreaUpdate() + task.run() + } + + def run(){ + def rets = findAllCity() + for(def city : rets){ + def retMap = findCityAndProvCode(city) + if(retMap){ + update(retMap) + } + } + } + + def update(retMap){ + def q = new BasicDBObject() + q.put("fangjia_city",retMap.fangjia_city) + q.put("district", "市辖区") + def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q) + if(obj){ + def udata = new BasicDBObject() + udata.put("city_code", retMap.city_code) + udata.put("province_code", retMap.province_code) + FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false) + } + } + + def findCityAndProvCode(def city){ + def q = new BasicDBObject() + q.put("fangjia_city",city) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def map = [:] + for(def d : dbcursor){ + if(d.city_code && d.province_code && d.district_code){ + map.put("fangjia_city",d.fangjia_city) + map.put("city_code",d.city_code) + map.put("province_code",d.province_code) + break + } + } + return map; + } + + def findAllCity(){ + def q = new BasicDBObject() + q.put("fangjia_district",null) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def citys = [] + for(def d : dbcursor){ + citys << d.fangjia_city + } + citys.unique() + return citys + } +} From ed0f3a05d413828ebaf0a553c84add80e610e1bd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 22 Jul 2021 09:59:18 +0800 Subject: [PATCH 002/111] =?UTF-8?q?=E6=8A=93=E6=97=A0=E9=94=A1=E6=9D=91?= =?UTF-8?q?=E5=BA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/core/util/BaiDuApiUtils.java | 3 +- .../BigdataCommbuildtypeBatch.groovy | 88 +++ .../BigdataCommunityBatch.groovy | 128 +++-- .../ccbevaluation/BigdataDealBatch.groovy | 416 ++++++++++++++ .../ccbevaluation/BigdataDistrictBatch.groovy | 6 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 11 +- .../ccbevaluation/BigdataForsaleBatch2.groovy | 521 ------------------ .../ccbevaluation/CcbCommonUtils.groovy | 51 +- .../script/ccbevaluation/CityBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 21 + .../script/gj/AddThirdPriceDistrict.groovy | 59 +- .../fangjia/script/gj/GaoDeMapCrawl.groovy | 11 +- .../script/gj/data/ExportDistrictInfo.groovy | 78 +++ 13 files changed, 772 insertions(+), 623 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index e075fbd32..582a71f23 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -258,8 +258,9 @@ public class BaiDuApiUtils { params.put("coords", lng.toString() + "," + lat.toString()); params.put("from", String.valueOf(from)); HttpRequest request = new HttpRequest(); + String result = null; try{ - String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); + result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); JSONObject json = JSONObject.fromObject(result); while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){ changeAK(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy new file mode 100644 index 000000000..93864962e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -0,0 +1,88 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 建行模型估值项目 + * 小区建筑类型数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataCommbuildtypeBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new BigdataCommbuildtypeBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rets = [] + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"]) + def rows = createDatas(cityCodeDatas, distCodeMap, dists) + rets.addAll(rows) + println city +" "+rows.size() + } + def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false); + println "txt 生成" + + } + + def createDatas(cityCodeDatas, distCodeMap, dists){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i> FITMENTS_TYPES_MAPS = new HashMap>() - static{ - FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) - FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) - FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) - FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) - } - - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) - - def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] - - static main(args) { - runBatch(new BigdataForsaleBatch2(), args); - } - - @Override - public void process(Object args) { - def citys = CcbCommonUtils.citys; - for(def city : citys){ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def distMap = getDistrictMap(city) - exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731); - } - } - - def getDistrictMap(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); - def distMap = [:] - for(def d : districts){ - distMap.put(d.region+"##"+d.name, d) - if(d.alias){ - d.alias.each{name-> - if(!distMap.containsKey(d.region+"##"+name)){ - distMap.put(d.region+"##"+name, d) - } - } - } - } - return distMap; - } - - def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){ - println city +" 小区key:"+distMap.size() - def cityCode = cityCodeDatas.city_code - def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap) - def allDatas = createSellDatas(city, retMap); - def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true); - println "txt文件生成 " - } - - - - def createSellDatas(def city, def map){ - - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(map.size()) - println city +"总房源量:"+map.size() - - def datas = [] - map.each{k,m-> - fixedThreadPool.execute(new Runnable() { - void run() { - try { - totalCount.addAndGet(1) - datas << converData(m) - println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString() - } catch (Exception e) { - e.printStackTrace() - } finally { - latch.countDown() - } - }; - }) - } - latch.await() - return datas ; - } - - def converData(def m){ - def data = [] - data << m.city_code - data << m.houseId - data << "" - data << m.city - data << m.region_code - data << m.region - data << m.district_id - data << m.district_name - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << m.address==null?"":m.address==null - data << m.build_year==null?"":m.build_year - data << m.proptype==null?"":m.proptype - data << m.propyears==null?"":m.propyears - data << m.avg_price==null?"":m.avg_price - data << m.total_price==null?"":m.total_price - data << "" - data << m.floor==null?"":m.floor - data << m.height==null?"":m.height - data << m.estate_type==null?"":m.estate_type - data << m.house_structure_code==null?"":m.house_structure_code - data << m.build_structure_code==null?"":m.build_structure_code - data << m.area==null?"":m.area - data << "" - data << m.house_type==null?"":m.house_type - data << m.room==null?"":m.room - data << m.hall==null?"":m.hall - data << "" - data << m.toilet==null?"":m.toilet - data << m.direction_code==null?"":m.direction_code - data << m.fitment_code==null?"":m.fitment_code - data << "" - data << "" - data << m.p_date==null?"":m.p_date - data << "" - data << "" - data << m.root - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << m.c_date==null?"":m.c_date - data << m.m_date==null?"":m.m_date - data << m.source - data << "" - return data - } - - - def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){ - - QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); - QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); - - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) - .must(date_range).must(area_range) - - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); - def retMap = new ConcurrentHashMap() - println city +" 没过滤前:"+ houses.size() - if(houses){ - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(houses.size()) - - for (int k=0;k60){ - return null - } - -// if(!seawater.room){ -// continue; -// } - if(!seawater.room){ - seawater.room = null - } - - if(seawater.room){ - if(seawater.room>=5 || seawater.room<1){ - return null - } - } - - if(!seawater.hall){ - seawater.hall = null - } - if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ - return null - } - if(!seawater.toilet){ - seawater.toilet = null - } - if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ - return null - } - - - def floor = parseFloor(seawater.floor_info) - if(seawater.height && floor && floor.isNumber()){ - if(floor.toInteger()>seawater.height){ - floor = null - seawater.height = null - } - - } - - if(!floor){ - seawater.floor = null - } - if(floor && floor.toInteger()<1){ - floor = null - } - - def rowMap = [:]; - if(cat == "rent"){ - rowMap.rent_type = seawater.rent_type - if(seawater.avg_price && seawater.avg_price>1000){ - return null - } - - if(seawater.avg_price && seawater.avg_price<10){ - return null - } - - } - rowMap.houseId = seawater.id - rowMap.city = city - //rowMap.region =seawater.region - //rowMap.address = seawater.address - - - rowMap.area = seawater.area - rowMap.total_price = seawater.total_price - rowMap.avg_price = seawater.avg_price - rowMap.height = seawater.height=="null"?null:seawater.height - if(rowMap.height && rowMap.height<1){ - rowMap.height = null - } - rowMap.floor = floor=="null"?null:floor - rowMap.room = seawater.room=="null"?null:seawater.room - rowMap.hall = seawater.hall=="null"?null:seawater.hall - rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet - if(rootMap.get(seawater.root)){ - rowMap.root = rootMap.get(seawater.root) - }else{ - rowMap.root = "99" - } - - def direction = parseDirection(seawater.direction) - rowMap.direction = direction=="null"?null:direction - - def fitment = parseFitment(seawater.fitment) - rowMap.fitment = fitment=="null"?null:fitment - - - - if(seawater.p_date instanceof String){ - seawater.p_date = seawater.p_date.toInteger() - } - if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ - rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") - rowMap.c_date = rowMap.p_date - rowMap.m_date = rowMap.p_date - }else{ - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) - } - - def regionCode = distCodeMap.get(rMap.region) - def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) - def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) - def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) - def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type); - def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); - rowMap.city_code = cityCode - rowMap.region = rMap.region - rowMap.region_code = regionCode - rowMap.district_id = rMap.get("_id").toString() - rowMap.district_name = rMap.name - rowMap.build_year = buildYear; - rowMap.proptype = housePro - rowMap.propyears = propertyYear; - rowMap.estate_type = estateType - rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall); - rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction) - rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment); - rowMap.address = rMap.address - rowMap.build_structure_code = struCode - rowMap.house_structure_code = null//getHouseStructureCode(); - rowMap.source = "FJW" - return rowMap - - } - - def getPublicDate(def pDate){ - - if(pDate>=20181001 && pDate<20200201){ - return DateUtils.getRandomDate("2018-10-01", "2020-02-01") - } - def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") - return pdate; - } - - static def parseFloor(def floorDesc){ - if(floorDesc){ - floorDesc = floorDesc.replaceAll("[\\s ]+", "") - if(floorDesc){ - Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); - Matcher matcher1 = pat1.matcher(floorDesc); - if (matcher1.find()) { - return matcher1.group(1) - } - pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); - matcher1 = pat1.matcher(floorDesc); - if (matcher1.find()) { - return matcher1.group(1) - } -// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ -// return "中层" -// } -// -// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ -// return "高层" -// } -// -// -// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ -// return "低层" -// } - if(floorDesc.isNumber()){ - return floorDesc; - } - } - } - return null; - } - - private String csvHandlerStr(String str) { - //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 - String tempDescription=str; - //如果有逗号 - if(str.contains(",")||str.contains(",")){ - //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 - if(str.contains("\"")){ -// tempDescription=str.replace("\"", "\"\""); - tempDescription=str.replace("\"", ""); - } - //在将逗号转义 - tempDescription="\""+tempDescription+"\""; - } - return tempDescription; - } - - - static String parseFitment(String fitmentDesc) { - if (fitmentDesc == null) { - return null - } - fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") - if (fitmentDesc.equals("非毛坯")) { - return null - } - Iterator>> it = FITMENTS_TYPES_MAPS - .entrySet().iterator() - Map.Entry> entry - while (it.hasNext()) { - entry = it.next() - if (fitmentDesc.contains(entry.getKey())) { - return entry.getKey() - } - for (String fitmentKey : entry.getValue()) { - if (fitmentDesc.contains(fitmentKey)) { - return entry.getKey() - } - } - } - return null - - } - - static String parseDirection(String directionDesc) { - if (directionDesc == null) { - return null - } - directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") - StringBuffer sb = new StringBuffer() - for (int i = 0; i < directionDesc.length(); i++) { - String c = directionDesc.substring(i, i + 1) - if (DIRECTIONS.contains(c)) { - if (directionDesc.substring(i).startsWith(c + "#_#")) { - sb.append(c + " ") - } else { - sb.append(c) - } - } - } - for (String d : DIRECTIONS) { - if (sb.toString().startsWith(d)) { - return d - } - } - for (String d : DIRECTIONS) { - if (sb.toString().contains(d)) { - return d - } - } - return null - - } - - /* - * 生成一个用不重复的ID - */ - def GenNonDuplicateID(randomLength){ - return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) - } - - - - def marge(){ - File file = new File("D:\\数据\\建行\\新建文本文档.txt"); - def ids = new HashSet(); - def index = 0 - - file.eachLine{it-> - index++ - if(index>1){ - def arrs = it.split("\\|@\\|") - def id = arrs[1] - if(!ids.contains(id)){ - ids << id - } - } - } - println index - println ids.size() - } - -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 37c99bc96..c11a06cfa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -7,8 +7,8 @@ import com.mongodb.DBCollection class CcbCommonUtils { - public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] -// public static def citys = ["北京"] + public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] +// public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -18,6 +18,11 @@ class CcbCommonUtils { public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static def findCityCodeValue(city){ def q = new BasicDBObject() q.put("fangjia_city", city) @@ -58,4 +63,46 @@ class CcbCommonUtils { } return rets } + + public static def findTradesDetail(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + def lists = [] + for(def d : dbcursor){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } + } + return lists + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy index bafa9e4d8..ef52b58c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -52,7 +52,7 @@ class CityBatch extends BaseBatch{ rets << headers rets.addAll(allDatas) ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8"); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy new file mode 100644 index 000000000..399e3c53a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -0,0 +1,21 @@ +package com.fangjia.script.ccbevaluation +import groovy.sql.Sql + +def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +def citys = CcbCommonUtils.citys; +def rows = [] + +for(def city : citys) { + def data = CcbCommonUtils.findCityCodeValue(city) + def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; + + def count = 0 + sqlInstance.execute(sql,{ + count = it.count + }); + println city +" "+count +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy index aece3ef15..d61bfb2cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy @@ -74,20 +74,20 @@ class AddThirdPriceDistrict { static main(args) { def task = new AddThirdPriceDistrict(); -// task.run() + task.run() - def mapName = "锦华星光苑3,4幢" +// def mapName = "锦华星光苑3,4幢" //// println task.pattern("港新花苑A1区"); - println mapName.replaceAll("\\w+-?\\w+(幢)", "") - println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") +// println mapName.replaceAll("\\w+-?\\w+(幢)", "") +// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") } def run(){ -// exportDatas(); + exportDatas(); // getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); // addInfo(); // addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx") // addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx") - addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); +// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); } def addCrawlMapInfo( def path){ @@ -135,29 +135,32 @@ class AddThirdPriceDistrict { def exportDatas(){ - def city = "无锡" - def datas = findMatchErrorDatas(city); - println "共【"+datas.size()+"】条" - datas = filterName(datas); - println "过滤后共【"+datas.size()+"】条" - def resMap = groupByName(datas); - def rets = [] - rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"] - resMap.each {k,v-> - def ret = [] - def arrs = k.split("##") - ret << arrs[0] - ret << arrs[1] - if(arrs.size()==3 && arrs[2]){ - ret << arrs[2] - }else{ - ret << "" + //跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞", + def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + for(def city : citys){ + def datas = findMatchErrorDatas(city); + println "共【"+datas.size()+"】条" + datas = filterName(datas); + println "过滤后共【"+datas.size()+"】条" + def resMap = groupByName(datas); + def rets = [] + rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"] + resMap.each {k,v-> + def ret = [] + def arrs = k.split("##") + ret << arrs[0] + ret << arrs[1] + if(arrs.size()==3 && arrs[2]){ + ret << arrs[2] + }else{ + ret << "" + } + ret << resAddrMap.get(k) + ret << v.join(",") + rets << ret } - ret << resAddrMap.get(k) - ret << v.join(",") - rets << ret + ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后") } - ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后") } def filterName(datas){ @@ -465,7 +468,7 @@ class AddThirdPriceDistrict { def findMatchErrorDatas(def city){ def q = new BasicDBObject() q.put("city",city) - q.put("s_date",202005) + q.put("s_date",new BasicDBObject('$gte', 202101)) // q.put("name","博济聚创科技园") // q.put("source","贝壳找房") def dbcursor = matchErrorLogColl.find(q) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 00c1ec513..a8b29aff0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -56,11 +56,11 @@ class GaoDeMapCrawl extends BaseBatch{ // def prefix = "road_" // def categorys = ["190301"]//道路相关 -// def prefix = "village_" -// def categorys = ["190108","190109"]//村庄 + def prefix = "village_" + def categorys = ["190108","190109"]//村庄 - def prefix = "company_" - def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] +// def prefix = "company_" +// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] def totalCount = new AtomicLong() static main(args) { @@ -118,7 +118,8 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" - def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys =["无锡"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy new file mode 100644 index 000000000..747ddf078 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + + +def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"] + +def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"] +def rets = [] +rets << headers + +for(def s : districts){ + def arrs = s.split("_") + def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2]) + def ret = [] + ret << s + if(m){ + ret << m.city + ret << m.district + ret << m.name + if(m.alias){ + ret << m.alias.toString().replace("[","").replace("]","") + }else{ + ret << "" + } + ret << m.address + if(m.addressAlias){ + ret << m.addressAlias.toString().replace("[","").replace("]","") + }else{ + ret << "" + } + + ret << m.avgPrice + ret << m.lng + ret << m.lat + ret << m.block + ret << m.loopLine + ret << m.propertyType + ret << m.buildingType + ret << m.commercialArea + ret << m.totalUnitCount + ret << m.managementFee + ret << m.parking + ret << m.parkingDescription + ret << m.ownershipPeriod + ret << m.ownershipType + ret << m.startBuildTime + ret << m.startSellingTime + ret << m.finishBuildTime + ret << m.moveinTime + ret << m.floorSpace + ret << m.constructionSpace + ret << m.plotRatio + ret << m.greenPercentage + ret << m.preSalePermit + ret << m.developer + ret << m.propertyManagement + ret << m.decorationStandard + ret << m.buildingStructure + ret << m.supportingFacilities + ret << m.transportation + ret << m.relatedInfo + ret << m.description + ret << m.salesOfficeAdd + ret << m.salesPhoneNumber + ret << m.discount + ret << m.buildingCount + ret << m.finishYear + + } + rets << ret +} +ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据") + + + + From be891b3229184cb5626b7194a3cf5335e9b96fe0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 22 Jul 2021 10:01:40 +0800 Subject: [PATCH 003/111] update --- .../fangjia/script/gj/EsSearchUtils.groovy | 2 +- .../com/fangjia/script/gj/FiledStrand.groovy | 1 + .../com/fangjia/script/gj/OpenApiUtils.groovy | 2 + .../fangjia/script/gj/WuXiAddressTest.groovy | 37 +++++++++++- .../fangjia/script/gj/YouDaDataParser.groovy | 6 +- .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 60 +++++++++++++++++++ .../fangjia/script/gj/ccb/FileUtils.groovy | 30 +++++----- .../script/gj/data/RegionPriceExport.groovy | 24 ++++++-- .../script/gj/district/ExportData.groovy | 46 +++++++------- .../gj/district/FangjiaClientUtils.groovy | 5 +- .../gj/district/MapDistrictProcess.groovy | 31 ++++++---- .../district/SeaweedDataMatchProcess.groovy | 13 ++-- .../script/gj/district/ZsPersonProcess.groovy | 2 +- .../script/gj/es/EsMonthHouseStat.groovy | 54 ++++++++--------- .../thirdPriceCrawl/ZhugefangCrawl4.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 1 + 16 files changed, 220 insertions(+), 96 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index dc7821343..4d50b3661 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -7,7 +7,7 @@ import java.sql.ResultSet class EsSearchUtils { public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1f5247429..3fd9abe32 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -394,6 +394,7 @@ class FiledStrand { } return estateCharege; } + static def processHouseholdCount(def household_count, def estateType2){ if(household_count){ household_count = household_count.toInteger() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 4ec26d7db..b5efa4f51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -29,6 +29,8 @@ class OpenApiUtils { rMap.region = dto.district rMap.name = dto.name rMap.id = dto.id + rMap.lng = dto.lng + rMap.lat = dto.lat if(dto.address && dto.address!="null"){ rMap.address = dto.address } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy index f8c8c0d5c..cbf9f5e12 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy @@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils class WuXiAddressTest { static main(args) { def task = new WuXiAddressTest() - task.run() + //task.run() + task.addAddress() + } + + def addAddress(){ + ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d-> + def prov = d['C'] + def city = d['D'] + def region = d['E'] + def address = d['B'] + def road = d['J'] + def street = d['R'] + + def newAddress = prov+"省"+city +"市" + if(region == "惠山"){ + newAddress = newAddress + region +"区" + }else{ + newAddress = newAddress + region +"市" + } + + d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "") + def dist = d['U'] + + def s1 = null + if(road){ + s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","") + newAddress = newAddress + street + dist + road + s1 + }else if(d['L']){ + s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","") + newAddress = newAddress + street + dist + d['L'] + s1 + } + + d['V'] = newAddress + },0) } def run(){ @@ -18,6 +51,8 @@ class WuXiAddressTest { // // } + + ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d-> if(!d['S']){ def locationStr = getLocationInfo(d['D'], d['Q']) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 0c2be14c8..d8ccfa02e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,11 +47,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-05-01" + static def statDate = "2021-06-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 89dff40d4..7a766b668 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -7,6 +7,13 @@ import java.util.regex.Pattern class CcbCodeUtils { + //否:0 是:1 + static def parseFlag(String flag){ + if(flag && flag == "是"){ + return 1; + } + return 0; + } static def parseBuildYear(String buildDate) { if(buildDate){ @@ -53,6 +60,44 @@ class CcbCodeUtils { return null } +// 高层塔楼 0101 增加“超高层” 0108 +// 高层板楼 0102 增加“老洋房” 0109 +// 小高层板楼 0103 增加“花园洋房” 0110 +// 多层板楼 0104 增加“四合院” 0111 +// 独栋别墅 0105 增加“老式别墅洋房” 0112 +// 联排别墅 0106 +// 叠拼别墅 0107 +// 其他建筑形式 0199 + + static def parseBuildType(def buildType){ + if(buildType){ + if(buildType.contains("老式别墅洋房")){ + return "0112" + }else if(buildType.contains("四合院")){ + return "0111" + }else if(buildType.contains("花园洋房")){ + return "0110" + }else if(buildType.contains("老洋房")){ + return "0109" + }else if(buildType.contains("叠拼")){ + return "0107" + }else if(buildType.contains("联排")){ + return "0106" + }else if(buildType.contains("独栋")){ + return "0105" + }else if(buildType.contains("多层")){ + return "0104" + }else if(buildType.contains("超高层")){ + return "0108" + }else if(buildType.contains("小高层")){ + return "0103" + }else if(buildType.contains("高层")){ + return "0102" + } + } + return "0199" + } + //朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399 static def getDirectionCode(def direction){ @@ -161,4 +206,19 @@ class CcbCodeUtils { } return 0; } + + + //房屋用途 + static def parseHouseUse(String houseUse){ + if(houseUse){ + if(houseUse.contains("住宅")){ + return 11; + }else if(houseUse.contains("商住")){ + return 13; + }else{ + return 99 + } + } + return 99 + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 09f80b9da..9382cb398 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -3,13 +3,13 @@ package com.fangjia.script.gj.ccb import org.apache.commons.lang.StringUtils class FileUtils { - static def batchCreateFile(ret, fileName, headers,charset){ + static def batchCreateFile(ret, fileName, headers,charset, isAppend){ File file = new File(fileName); - if (file.exists()) { - file.delete(); - } +// if (file.exists()) { +// file.delete(); +// } - OutputStream out = new FileOutputStream(fileName); + OutputStream out = new FileOutputStream(fileName, isAppend); def sb = new StringBuffer(); sb.append(headers.join("|@|")) @@ -20,18 +20,18 @@ class FileUtils { def index = 0 for(def row : ret){ - index++ - sb.append(StringUtils.join(row.toArray(), "|@|")) - sb.append(System.getProperty("line.separator")); - println index + if(row){ + index++ + sb.append(StringUtils.join(row.toArray(), "|@|")) + sb.append(System.getProperty("line.separator")); - if(index%10000==0){ - println index+"开始写入CSV "+new Date().toLocaleString() - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - println index+"结束写入CSV "+new Date().toLocaleString() + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } } - } if(sb.length()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy index d059de4ae..a56e2079d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.data import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -10,22 +11,23 @@ import com.mongodb.DBCollection class RegionPriceExport { DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com") + DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") static main(args) { def task = new RegionPriceExport(); task.run() } def run(){ - def months = [202010,202011,202012,202101,202102,202103] - def regions = CommonUtils.getAllRegions("福州","region") + def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + def regions = CommonUtils.getAllRegions("佛山","region") def rets = [] - rets << ["城市","区域","202010","202011","202012","202101","202102","202103"] + rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] regions.each{region,values-> def data = [] - data << "福州" + data << "佛山" data << region for(def month :months){ - def dbo = findPrice("福州", region, month); + def dbo = findWeekendPrice("佛山", region, month); if(dbo){ data << dbo.avg_price }else{ @@ -34,7 +36,7 @@ class RegionPriceExport { } rets << data } - ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格") + ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格") } def findPrice(city, region, month){ @@ -45,4 +47,14 @@ class RegionPriceExport { def dbo = REGION_SELL_STAT.findOne(q) return dbo } + + def findWeekendPrice(city, region, month){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("cat", "区域") + q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) + def dbo = REGION_STAT.findOne(q) + return dbo + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index 6a44d4139..6f9219576 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.gj.district import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject @@ -13,7 +15,7 @@ class ExportData { def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol(); public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -24,8 +26,8 @@ class ExportData { static main(args) { def task = new ExportData(); - task.exportThird() -// task.exportMap(); +// task.exportThird() + task.exportMap(); } def exportMap(){ @@ -51,19 +53,18 @@ class ExportData { //"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特" //宁夏 ----- 中卫 固原 石嘴山 吴忠 银川 //青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁 - def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州", - "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮", - "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"] - //"沈阳","大连","抚顺","鞍山" - //"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆" - //"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛", - //"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州", - //"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城", +// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州", +// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮", +// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"] + def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]; + + for(def city : citys){ + def path = "D:\\数据\\小区\\地图\\${city}.xlsx"; + def datas = findAllDatas(city, null) + println "${city}共计:"+datas.size() + ExcelUtils.write(path, datas, "数据") + } - def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx"; - def datas = findAllDatas(citys, null) - println "共计:"+datas.size() - ExcelUtils.write(path, datas, "数据") } @@ -98,9 +99,9 @@ class ExportData { - def findAllDatas(def citys, def pname){ + def findAllDatas(def city, def pname){ final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1") + sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'") // sqlBuffer.append(" and pname = '"+pname+"'") // sqlBuffer.append(" and cityname = '"+pname+"'") ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) @@ -113,16 +114,19 @@ class ExportData { if(source && source=="高德"){ println index def type = rs.getObject("type") - def city = rs.getString("cityname") - if(citys.contains(city) && type.contains("商务住宅")){ + //def city = rs.getString("cityname") + if(type.contains("住宅")){ def ret = [] ret << city ret << rs.getString("adname") ret << rs.getString("name") ret << rs.getString("address") ret << rs.getString("type") - ret << rs.getFloat("location.lon") - ret << rs.getFloat("location.lat") + def loaction = rs.getObject("location") + if(loaction){ + ret << DataHandleUtils.getDoubleRound(6, loaction.x) + ret << DataHandleUtils.getDoubleRound(6, loaction.y) + } rets << ret } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 8eacf492b..ac634e9e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -68,13 +68,16 @@ class FangjiaClientUtils { public static def addressStand(def address){ def jsonObj = null + def content = null try { + address = address.replace("{","") def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") - def content = new HttpRequest().newGet(url, null, "utf-8") + content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) } catch (Exception e) { + println address +" "+content e.printStackTrace(); } return jsonObj; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy index 8f5809bed..f53276b72 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy @@ -1,9 +1,12 @@ package com.fangjia.script.gj.district +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DataUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation import java.util.regex.Pattern @@ -20,20 +23,13 @@ class MapDistrictProcess { * @return */ def run(){ - //"江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx","安徽地图.xlsx", - //"上海地图.xlsx","北京地图.xlsx" -//"安徽地图.xlsx","江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx", - //"重庆地图.xlsx","天津地图.xlsx", - //"上海地图.xlsx","北京地图.xlsx", - //"河北地图.xlsx","河南地图.xlsx","湖北地图.xlsx","湖南地图.xlsx","广西地图.xlsx", - //"吉林地图.xlsx","辽宁地图.xlsx""辽宁地图.xlsx","吉林地图.xlsx","黑龙江地图.xlsx","山西地图.xlsx","海南地图.xlsx"] - def files = ["青海地图.xlsx","宁夏地图.xlsx","内蒙古地图.xlsx","云南地图.xlsx","陕西地图.xlsx","新疆地图.xlsx","四川地图.xlsx"] + def files = ["武汉.xlsx"] for(def fileName : files){ def filePath = "D:\\数据\\小区\\地图\\"+fileName; - typeFilter(filePath) - nameStandard(filePath); - dataFilter(filePath) +// typeFilter(filePath) +// nameStandard(filePath); +// dataFilter(filePath) matchDistDatas(filePath) houseCalculate(filePath); ////// //统计是否存在多个区域 @@ -114,15 +110,24 @@ class MapDistrictProcess { def matchDistDatas(def path){ def index = 0 - ExcelUtils.multiThreadingModify(path,{ List row -> + ExcelUtils.modify(path,{ List row -> index ++ if(!row['I'] && !row['J']){ - def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],); + def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']); if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){ row['J'] = rMap.city row['K'] = rMap.region row['L'] = rMap.name row['M'] = rMap.address + + if(rMap.lat){ + def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3) + def maplng = baidu.x + def maplat = baidu.y + row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng) + def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L']) + row['Y'] = score + } } println index +" "+row } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 6a5c21a8e..5b0910fc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -26,15 +26,18 @@ class SeaweedDataMatchProcess { } } - if(!rMap && address){ -// if (findPattern(address)){ - rMap = OpenApiUtils.matchDistrict(city, address, region, true) -// } - } if(!rMap && name){ rMap = OpenApiUtils.matchDistrict(city, name, region, false) } + if(!rMap && address){ + rMap = OpenApiUtils.matchDistrict(city, address, region, true) + if(rMap && rMap.region && !region.contains(rMap.region)){ + rMap = null + } + } + + if(rMap && rMap.id && region.contains(rMap.region)) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy index e760b169f..cfa92f44a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy @@ -49,7 +49,7 @@ class ZsPersonProcess { // def loactionMap = task.matchMap(dist) // println loactionMap -// task.matchMap() + task.matchMap() // task.matchDb() // task.matchDb2() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index d8c7e72a3..be10cf533 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,9 +19,9 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() -// task.houseRun() + task.houseRun() - task.statHouseUpdateCount() +// task.statHouseUpdateCount() } def run(){ @@ -99,37 +99,35 @@ class EsMonthHouseStat { } def houseRun(){ - ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d-> - def city = d['B'] - if(city == "湖州"){ - def count = findHouseCount(city, 20210301, 20210601) - d['E'] = count + ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d-> + def city = d['A'] + def count = findHouseCount(city, 20190709, 20210709) + d['E'] = count - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); - def disthouseCount = 0 - for(def dist : districts){ - def alias = [] - if(dist.alias){ - if(!dist.alias.contains(dist.name)){ - dist.alias << dist.name - } - alias = dist.alias - }else{ - alias << dist.name + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def disthouseCount = 0 + for(def dist : districts){ + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name } - - def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601) - if(distCount){ - disthouseCount++ - } - println dist.city+" "+dist.region+" "+dist.name+" "+distCount + alias = dist.alias + }else{ + alias << dist.name } - println "disthouseCount:"+disthouseCount - println districts.size() - def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) - d['D'] = precent+"%" + def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709) + if(distCount){ + disthouseCount++ + } + println dist.city+" "+dist.region+" "+dist.name+" "+distCount } + + println "disthouseCount:"+disthouseCount + println districts.size() + def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) + d['F'] = precent+"%" },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy index c697657e8..a57c51cc7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy @@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{ @Override void process(Object args) { - citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""]) + citys.addAll(["福州"]) // getNoDataCitys(); def cityurl = "http://www.zhuge.com/areachange/" def cityUrls = getCityUrlCrawal(cityurl); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 6e5696107..841fb2f08 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -140,6 +140,7 @@ public class OpenApiClient { return null } + public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) { if (StringUtils.isBlank(input)) { From f4396610b3f3637aa9eee52b34ba5db307080182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 23 Jul 2021 10:59:21 +0800 Subject: [PATCH 004/111] =?UTF-8?q?=E9=A3=8E=E9=99=A9=E8=AF=84=E7=BA=A7?= =?UTF-8?q?=E5=9F=8E=E5=B8=82=E9=9D=9E=E5=BF=85=E5=A1=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/AddressController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index ffd9e0b9a..953611a47 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -2751,10 +2751,10 @@ public class AddressController { @Authority(role = Role.USER) public Object riskAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); - if (StringUtils.isEmpty(model.getCity())) { - mav.setModel(BasicModel.buildBadRequestModel("参数错误,城市不能为空!")); - return mav; - } +// if (StringUtils.isEmpty(model.getCity())) { +// mav.setModel(BasicModel.buildBadRequestModel("参数错误,城市不能为空!")); +// return mav; +// } if (StringUtils.isEmpty(model.getAddress())) { mav.setModel(BasicModel.buildBadRequestModel("参数错误,地址不能为空!")); return mav; From 527b069cf709eeffac0255cfdfc38a2a6e8ad1b0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 30 Jul 2021 17:10:38 +0800 Subject: [PATCH 005/111] =?UTF-8?q?=E6=B5=99=E5=95=86=E9=A3=8E=E9=99=A9?= =?UTF-8?q?=E7=BC=93=E9=87=8A=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/CzbankController.java | 484 ++++++++++++++---- .../fangjia/open/model/CzbankSampleModel.java | 166 ++++++ .../fangjia/open/ws/CzbankServiceImpl.java | 6 +- 3 files changed, 552 insertions(+), 104 deletions(-) create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/model/CzbankSampleModel.java diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/CzbankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/CzbankController.java index 88a0c3108..9649eff7b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/CzbankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/CzbankController.java @@ -1,154 +1,424 @@ package com.fangjia.open.controllers; +import com.fangjia.core.util.AnalyticsUtil; +import com.fangjia.core.util.BeanUtils; import com.fangjia.core.util.DataUtil; import com.fangjia.dal.common.DistrictKey; +import com.fangjia.dal.mongo.model.CzbankSampleLog; +import com.fangjia.dal.mongo.model.Evaluation; +import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap; import com.fangjia.dal.seaweed.model.District; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.CzbankSampleModel; import com.fangjia.open.model.RestfulModelAndView; +import com.fangjia.open.util.EvaluationFactorUtil; +import com.fangjia.open.ws.query.CollateralSampleQuery; +import com.fangjia.open.ws.query.EvaluationQuery; +import com.fangjia.service.api.AddressMatchProxyService; import com.fangjia.service.bo.Collateral; +import com.fangjia.service.bo.EvaluationResult; +import com.fangjia.service.bo.Sample; import com.fangjia.service.datastore.CzbankSampleService; import com.fangjia.service.datastore.DistrictService; +import com.fangjia.service.datastore.EvaluationService; import com.fangjia.service.datastore.bean.CollateralSampleQueryBean; +import com.fangjia.service.mongo.fangjia.CzbankSampleLogService; +import com.fangjia.service.open.FangjiaOfficialAreaService; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import net.sf.json.JsonConfig; +import net.sf.json.util.PropertyFilter; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; +import org.apache.cxf.interceptor.Fault; import org.dom4j.Document; import org.dom4j.DocumentHelper; import org.dom4j.Element; +import org.sothis.web.mvc.ActionContext; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /** * 浙商银行风险缓释项目接口 */ +@Authority(role = Role.USER) public class CzbankController { private CzbankSampleService czbankSampleService; private DistrictService districtService; + private CzbankSampleLogService czbankSampleLogService; + private FangjiaOfficialAreaService fangjiaOfficialAreaService; + private AddressMatchProxyService addressMatchProxyService; + private EvaluationService evaluationService; + + private List PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02"); /** * 样本查询接口 - * @param request + * @param model * @return */ - public Object sampleAction(HttpServletRequest request){ + @Authority(role = Role.USER) + public Object sampleAction(CzbankSampleModel model){ RestfulModelAndView mav = new RestfulModelAndView(); - List queryBeans = null; - try { - queryBeans = parseParams(request); - } catch (Exception e) { - mav.setModel(BasicModel.buildBadRequestModel(e.getMessage())); + + long start = System.currentTimeMillis(); + if(StringUtils.isBlank(model.getCltNo())){ + mav.setModel(BasicModel.buildBadRequestModel("cltNo押品编号不能为空")); return mav; } - if(queryBeans == null || queryBeans.isEmpty()){ - mav.setModel(BasicModel.buildBadRequestModel("参数无效")); + + if (StringUtils.isBlank(model.getProvince()) ||StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getDistrict()) + || StringUtils.isBlank(model.getBuildingName()) || StringUtils.isBlank(model.getPropertyTypeCode())) { + mav.setModel(BasicModel.buildBadRequestModel("参数缺失:province(省),city(城市),district(区域),buildingName(名称),propertyTypeCode(物业类型编码)是必须参数")); return mav; } - String paramMax = request.getParameter("max"); - if(StringUtils.isBlank(paramMax) || !StringUtils.isNumeric(paramMax)){ - paramMax = "5"; + + FangjiaOfficialAreaMap fangjiaOfficialAreaMap = fangjiaOfficialAreaService.convertOfficialCodeToFangjia(model.getDistrict()); + if(fangjiaOfficialAreaMap == null){ + fangjiaOfficialAreaMap = fangjiaOfficialAreaService.convertOfficialCodeToFangjiaCity(model.getDistrict()); } - int max = Integer.valueOf(paramMax); - if(max < 1 || max > 20){ - max = 5; + if(fangjiaOfficialAreaMap == null){ + mav.setModel(BasicModel.buildBadRequestModel("地区信息无法映射到房价网的地区信息->(province:" + model.getProvince() + " ,city: " + model.getCity() + " ,district: " + model.getDistrict() + ")")); + return mav; } - List collateralList = czbankSampleService.findSample(queryBeans, max); - mav.setModel(BasicModel.buildOKModel().setResult(collateralList)); + + + + String propertyTypeCode = model.getPropertyTypeCode(); + if(StringUtils.isBlank(propertyTypeCode)){ + propertyTypeCode = "01"; + } + if(!PROPERTY_TYPE_CODE_LIST.contains(propertyTypeCode)){ + mav.setModel(BasicModel.buildBadRequestModel("propertyTypeCode取值非法")); + return mav; + } + if(model.getStrctureArea() <= 0){ + mav.setModel(BasicModel.buildBadRequestModel("strctureArea取值非法")); + return mav; + } + + + if (model.getFloorNumber() != null && (model.getFloorNumber() <= 0 || model.getFloorNumber() > 59)) { + mav.setModel(BasicModel.buildBadRequestModel("floorNumber,正确范围[1,60)")); + return mav; + } + + if (model.getAboveFloorAmount() != null && (model.getAboveFloorAmount() <= 0 || model.getAboveFloorAmount() > 59)) { + mav.setModel(BasicModel.buildBadRequestModel("floorAmount,正确范围[1,60)")); + return mav; + } + if (model.getAboveFloorAmount() != null && model.getAboveFloorAmount() != null && model.getFloorNumber()> model.getAboveFloorAmount()) { + mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层")); + return mav; + } + + + + Integer decrCondCode = model.getDecrCondCode(); + if(decrCondCode == null){ + decrCondCode = 2; + } + if(decrCondCode > 5 || decrCondCode < 1){ + mav.setModel(BasicModel.buildBadRequestModel("decrCondCode取值非法")); + return mav; + } + + Integer towardDirectCode = model.getTowardDirectCode(); + if(towardDirectCode == null || towardDirectCode == 0){ + towardDirectCode = 3; + } + if(towardDirectCode > 10 || towardDirectCode < 1){ + mav.setModel(BasicModel.buildBadRequestModel("towardDirectCode取值非法")); + return mav; + } + CollateralSampleQueryBean queryBean = new CollateralSampleQueryBean(); + BeanUtils.copyProperties(model, queryBean); + queryBean.setFangjiaCity(fangjiaOfficialAreaMap.getFangjiaCity()); + queryBean.setFangjiaDistrict(fangjiaOfficialAreaMap.getFangjiaDistrict()); + + District property = districtService.findByNameAndAlias(queryBean.getFangjiaCity(), queryBean.getFangjiaDistrict(), model.getBuildingName(), "district"); + if(property == null){ + Map params = new HashMap(); + params.put("city", queryBean.getFangjiaCity()); + params.put("suggestDistrict", queryBean.getFangjiaDistrict()); + if(StringUtils.isBlank(queryBean.getAddress())){ + params.put("address", model.getBuildingName()); + }else{ + params.put("address", queryBean.getAddress()); + params.put("suggestName", model.getBuildingName()); + } + params.put("category", "district"); + String result = addressMatchProxyService.parser(params); + if (result != null) { + JSONObject json = JSONObject.fromObject(result); + if(json.getInt("code")==200){ + Map map = (Map) json.get("result"); + if((Integer)map.get("precise")>=0){ + property = districtService.findByNameAndAlias((String) map.get("city"), (String) map.get("district"), (String) map.get("name"), "district"); + } + } + } + if(property == null){ + mav.setModel(BasicModel.buildBadRequestModel("该楼盘不存在 city:" + queryBean.getFangjiaCity() + " region:" + queryBean.getFangjiaDistrict() + " name:" + model.getBuildingName())); + return mav; + } + } + + queryBean.setProperty(property); + List queryBeans = new ArrayList(); + queryBeans.add(queryBean); + List collaterals = czbankSampleService.findSample(queryBeans, 5); + long time = System.currentTimeMillis() - start; + Map stat = new HashMap(); + for(Collateral collateral : collaterals){ + // stat.put(collateral.getCltNo(), collateral.getSample()); + stat.put("cltNo", collateral.getCltNo()); + stat.put("sample", collateral.getSample()); + } + for(CollateralSampleQueryBean qb : queryBeans){ + JsonConfig jsonConfig = new JsonConfig(); + jsonConfig.setJsonPropertyFilter(new PropertyFilter() { + @Override + public boolean apply(Object o, String s, Object o1) { + return o1 == null; + } + }); + qb.setProperty(null); + JSONObject query = JSONObject.fromObject(qb, jsonConfig); + CzbankSampleLog log = new CzbankSampleLog(); + log.setCaseId(qb.getCltNo()); + log.setCreatedDate(new Date()); + log.setQuery(DataUtil.toMongoDbObj(query)); + List samples = (List)stat.get("sample"); + if(samples != null){ + log.setSamples(DataUtil.toMongoDbObj(JSONArray.fromObject(samples, jsonConfig))); + } + log.setTime(time); + czbankSampleLogService.save(log); + } + mav.setModel(BasicModel.buildOKModel().setResult(stat)); return mav; } - private List parseParams(HttpServletRequest request) throws Exception { - String xml = null; + + @Authority(role = Role.USER) + public Object evaluationAction(EvaluationQuery evaluationQuery) { + RestfulModelAndView mav = new RestfulModelAndView(); + District property = null; try { - xml = IOUtils.toString(request.getInputStream(), "UTF-8"); + if (StringUtils.isBlank(evaluationQuery.getCity()) || StringUtils.isBlank(evaluationQuery.getDistrict()) || StringUtils.isBlank(evaluationQuery.getName())) { + mav.setModel(BasicModel.buildBadRequestModel("参数缺失:city(城市),district(区域),buildingName(名称)是必须参数")); + return mav; + } + + if(evaluationQuery.getSize() <= 0){ + mav.setModel(BasicModel.buildBadRequestModel("size不能为空")); + return mav; + } + + if (evaluationQuery.getCarportPrice() != null && (evaluationQuery.getCarportPrice() < 0 || evaluationQuery.getCarportPrice() > 1000000)) { + mav.setModel(BasicModel.buildBadRequestModel("carportPrice(车位)取值不合法")); + return mav; + } + + if (evaluationQuery.getMaxFloor() != null && (evaluationQuery.getMaxFloor() <= 0 || evaluationQuery.getMaxFloor() > 59)) { + mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)")); + return mav; + } + + if (evaluationQuery.getFloor() != null && (evaluationQuery.getFloor() <= 0 || evaluationQuery.getFloor() > 59)) { + mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)")); + return mav; + } + if (evaluationQuery.getMaxFloor() != null && evaluationQuery.getFloor() != null && evaluationQuery.getFloor() > evaluationQuery.getMaxFloor()) { + mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层")); + return mav; + } + + FangjiaOfficialAreaMap fangjiaOfficialAreaMap = fangjiaOfficialAreaService.convertOfficialCodeToFangjia(evaluationQuery.getDistrict()); + if(fangjiaOfficialAreaMap == null){ + fangjiaOfficialAreaMap = fangjiaOfficialAreaService.convertOfficialCodeToFangjiaCity(evaluationQuery.getDistrict()); + } + if(fangjiaOfficialAreaMap == null){ + mav.setModel(BasicModel.buildBadRequestModel("地区信息无法映射到房价网的地区信息->(province:" + evaluationQuery.getProvince() + " ,city: " + evaluationQuery.getCity() + " ,district: " + evaluationQuery.getDistrict() + ")")); + return mav; + } + String fangjiaCity = fangjiaOfficialAreaMap.getFangjiaCity(); + String fangjiaDistrict = fangjiaOfficialAreaMap.getFangjiaDistrict(); + + property = districtService.findByNameAndAlias(fangjiaCity, fangjiaDistrict, evaluationQuery.getName(), "district"); + if(property == null){ + Map params = new HashMap(); + params.put("city", fangjiaCity); + params.put("suggestDistrict", fangjiaDistrict); + if(StringUtils.isBlank(evaluationQuery.getAddress())){ + params.put("address", evaluationQuery.getName()); + }else{ + params.put("address", evaluationQuery.getAddress()); + params.put("suggestName", evaluationQuery.getName()); + } + params.put("category", "district"); + String result = addressMatchProxyService.parser(params); + if (result != null) { + JSONObject json = JSONObject.fromObject(result); + if(json.getInt("code")==200){ + Map map = (Map) json.get("result"); + if((Integer)map.get("precise")>=0){ + property = districtService.findByNameAndAlias((String) map.get("city"), (String) map.get("district"), (String) map.get("name"), "district"); + } + } + } + + if(property == null){ + mav.setModel(BasicModel.buildBadRequestModel("该楼盘不存在 city:" + fangjiaCity + " region:" + fangjiaDistrict + " name:" + evaluationQuery.getName())); + return mav; + } + + } + + if(property != null){ + if(StringUtils.isNotBlank(fangjiaDistrict) && !property.getRegion().equals(fangjiaDistrict)){ + mav.setModel(BasicModel.buildBadRequestModel("该楼盘不存在 city:" + fangjiaCity + " region:" + fangjiaDistrict + " name:" + evaluationQuery.getName())); + return mav; + } + } } catch (Exception e) { - - } - if(StringUtils.isBlank(xml)){ - throw new Exception("数据读取异常"); - } - Document document; - try{ - document = DocumentHelper.parseText(xml); - }catch (Exception e){ - throw e; + throw new Fault(e); } + EvaluationResult result = null; + if (property != null) { + HttpServletRequest request = ActionContext.getContext().getRequest(); + Evaluation evaluation = evaluationService.evaluate(property, buildEvaluation(evaluationQuery, request), + evaluationQuery.getCalDate()); - - Element root = document.getRootElement(); - List params = root.elements(); - List queryBeans = new ArrayList(); - for(Element param : params){ - - String cltNo = param.elementText("cltNo"); - if(StringUtils.isBlank(cltNo)){ - throw new Exception("cltNo不能为空"); + if(null != evaluation && evaluation.getTotalPrice()!=null){ + result = new EvaluationResult(); + result.setTotalPrice(evaluation.getTotalPrice()); + result.setAvgPrice(evaluation.getAvgPrice()); + }else{ + mav.setModel(BasicModel.buildBadRequestModel("该楼盘暂无法估值!")); + return mav; } - String city = param.elementText("city"); - if(StringUtils.isBlank(city)){ - throw new Exception("city不能为空,cltNo:" + cltNo); - } - String district = param.elementText("district"); - if(StringUtils.isBlank(district)){ - throw new Exception("district不能为空,cltNo:" + cltNo); - } - String buildingName = param.elementText("buildingName"); - if(StringUtils.isBlank(buildingName)){ - throw new Exception("buildingName不能为空,cltNo:" + cltNo); - } - District d = districtService.findOneByKey(new DistrictKey(city, district, buildingName), "district"); - if(d == null){ - throw new Exception("楼盘不存在,cltNo:" + cltNo); - } - String propertyTypeCode = param.elementText("propertyTypeCode"); - if(StringUtils.isBlank(propertyTypeCode)){ - throw new Exception("propertyTypeCode不能为空,cltNo:" + cltNo); - } - if(!"1".equals(propertyTypeCode) && !"0".equals(propertyTypeCode)){ - throw new Exception("propertyTypeCode取值非法,cltNo:" + cltNo); - } - String floorNumber = param.elementText("floorNumber"); - if(StringUtils.isBlank(floorNumber) || !StringUtils.isNumeric(floorNumber)){ - throw new Exception("floorNumber取值非法,cltNo:" + cltNo); - } - String strctureArea = param.elementText("strctureArea"); - if(StringUtils.isBlank(strctureArea) || !DataUtil.isNumeric(strctureArea)){ - throw new Exception("strctureArea取值非法,cltNo:" + cltNo); - } - String decrCondCode = param.elementText("decrCondCode"); - if(StringUtils.isBlank(decrCondCode)){ - decrCondCode = "2"; - } - if(!StringUtils.isNumeric(decrCondCode) || (Integer.valueOf(decrCondCode) > 5 || Integer.valueOf(decrCondCode) < 1)){ - throw new Exception("decrCondCode取值非法,cltNo:" + cltNo); - } - String towardDirectCode = param.elementText("towardDirectCode"); - if(StringUtils.isBlank(towardDirectCode)){ - towardDirectCode = "3"; - } - if(!StringUtils.isNumeric(towardDirectCode) || (Integer.valueOf(towardDirectCode) > 10 || Integer.valueOf(towardDirectCode) < 1)){ - throw new Exception("towardDirectCode取值非法,cltNo:" + cltNo); - } - CollateralSampleQueryBean queryBean = new CollateralSampleQueryBean(); - queryBean.setCltNo(cltNo); - queryBean.setCity(city); - queryBean.setDistrict(district); - queryBean.setBuildingName(buildingName); - queryBean.setPropertyTypeCode(propertyTypeCode); - queryBean.setFloorNumber(Integer.valueOf(floorNumber)); - queryBean.setStrctureArea(Double.valueOf(strctureArea)); - queryBean.setTowardDirectCode(Integer.valueOf(towardDirectCode)); - queryBean.setDecrCondCode(Integer.valueOf(decrCondCode)); - queryBean.setProperty(d); - queryBeans.add(queryBean); } - return queryBeans; + mav.setModel(BasicModel.buildOKModel().setResult(result)); + return mav; } + private Evaluation buildEvaluation(EvaluationQuery model, HttpServletRequest request) { + Evaluation evaluation = new Evaluation(); + evaluation.setCity(model.getCity()); + evaluation.setRegion(model.getDistrict()); + evaluation.setDistrictName(model.getName()); + + + if (StringUtils.isNotBlank(model.getBuildingNumber())) { + evaluation.setHouseNum(model.getBuildingNumber()); + } + + if (StringUtils.isNotBlank(model.getRoomNumber())) { + evaluation.setRoomNum(model.getRoomNumber()); + } + evaluation.setArea(model.getSize()); + + if (null != model.getFloor()) { + evaluation.setFloor(model.getFloor()); + } + if (model.getMaxFloor() != null && 0 != model.getMaxFloor()) { + evaluation.setHeight(model.getMaxFloor()); + } + if (0 != model.getRoom()) { + evaluation.setRoom(model.getRoom()); + } + if (0 != model.getHall()) { + evaluation.setHall(model.getHall()); + } + if (0 != model.getToilet()) { + evaluation.setToilet(model.getToilet()); + } + evaluation.setEstateType(EvaluationFactorUtil.PROPERTY_TYPES.get(model.getPropertyType())); + evaluation.setDirection(EvaluationFactorUtil.DIRECTION.get(model.getDirection())); + evaluation.setFitment(EvaluationFactorUtil.EVALUATION_MODEL_FITMENT.get(EvaluationFactorUtil.FITMENT.get(model.getDecoration()))); + if (model.getDecorationAge() >= 1 && model.getDecorationAge() <= 5) { + evaluation.setFitmentAge(model.getDecorationAge()); + } + + if (model.getDecorationCost() != null && model.getDecorationCost() > 0) { + evaluation.setFitmentCharge(model.getDecorationCost()); + } + evaluation.setSight(EvaluationFactorUtil.SIGHT.get(model.getView())); + evaluation.setNoise(EvaluationFactorUtil.NOISE.get(model.getNoise())); + + if (model.getGardenSize()!=null && 0 != model.getGardenSize()) { + evaluation.setGardenArea(((Number) model.getGardenSize()).doubleValue()); + } + if (model.getBasementSize()!=null && 0 != model.getBasementSize()) { + evaluation.setBasementArea(((Number) model.getBasementSize()).doubleValue()); + } + + if (model.getAtticSize()!=null && 0 != model.getAtticSize()) { + evaluation.setAtticArea(((Number) model.getAtticSize()).doubleValue()); + } + +// if (0 != model.getOuterSize()) { +// evaluation.setOtherFactor(((Number) model.getOuterSize()).doubleValue()); +// } + + String worryFacility = model.getNegativeFactor(); + evaluation.setWorryFacility(EvaluationFactorUtil.NEGATIVE_FACTOR.get(worryFacility)); + if (0 != model.getAge()) { + evaluation.setAge(model.getAge()); + } + evaluation.setCarportPrice(model.getCarportPrice()); + + if (model.getFacilities() != null) { + String facilities = model.getFacilities(); + String[] fs = facilities.split(","); + for (int i = 0; i < fs.length; i++) { + fs[i] = EvaluationFactorUtil.FACILITIES.get(fs[i]); + } + evaluation.setFacilities(convertToString(fs)); + } + + evaluation.setCreationDate(new Date()); + evaluation.setModificationDate(new Date()); + + + String ip = AnalyticsUtil.getIpAddress(request); + evaluation.setIp(ip); + + String fromUserId = (String) request.getAttribute("from_user_id"); + if (StringUtils.isNotBlank(fromUserId)) { + evaluation.setFromUserId(String.valueOf(fromUserId)); + } + evaluation.setSource("cz_property_evaluation"); + return evaluation; + } + + private String convertToString(String[] strs) { + if (ArrayUtils.isNotEmpty(strs)) { + StringBuffer ret = new StringBuffer(); + for (String str : strs) { + if (StringUtils.isEmpty(ret.toString())) { + ret.append(str); + } else { + ret.append(",").append(str); + } + } + return ret.toString(); + } + return null; + } + + + public void setCzbankSampleService(CzbankSampleService czbankSampleService) { this.czbankSampleService = czbankSampleService; } @@ -156,4 +426,20 @@ public class CzbankController { public void setDistrictService(DistrictService districtService) { this.districtService = districtService; } + + public void setCzbankSampleLogService(CzbankSampleLogService czbankSampleLogService) { + this.czbankSampleLogService = czbankSampleLogService; + } + + public void setFangjiaOfficialAreaService(FangjiaOfficialAreaService fangjiaOfficialAreaService) { + this.fangjiaOfficialAreaService = fangjiaOfficialAreaService; + } + + public void setAddressMatchProxyService(AddressMatchProxyService addressMatchProxyService) { + this.addressMatchProxyService = addressMatchProxyService; + } + + public void setEvaluationService(EvaluationService evaluationService) { + this.evaluationService = evaluationService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/CzbankSampleModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/CzbankSampleModel.java new file mode 100644 index 000000000..a24154823 --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/CzbankSampleModel.java @@ -0,0 +1,166 @@ +package com.fangjia.open.model; + +import org.hibernate.validator.constraints.NotBlank; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.util.Date; + +public class CzbankSampleModel extends FangjiaApiModel{ + private String cltNo; + private String province; + /**城市*/ + private String city; + /**区域*/ + private String district; + /**小区名*/ + private String buildingName; + /**楼栋名称*/ + private String buildingNoName; + /**物业类型编码*/ + private String propertyTypeCode; + /**是否有电梯 0:无 1:有*/ + private Integer isElevator; + /**总楼层*/ + private Integer aboveFloorAmount; + /**所在楼层*/ + private Integer floorNumber; + /** + * 装修情况编码 + * 1:毛坯 + * 2:简装(默认) + * 3:精装 + * 4:豪装 + * */ + private Integer decrCondCode; + /**面积**/ + private Double strctureArea; + /** + * 朝向情况编码 + * 1:西南 + * 2:东南 + * 3:南(默认) + * 4:西北 + * 5:东北 + * 6:东 + * 7:西 + * 8:北 + * 9:东西 + * 10:南北 + * */ + private Integer towardDirectCode; + private Date updateDate; + + public String getCltNo() { + return cltNo; + } + + public void setCltNo(String cltNo) { + this.cltNo = cltNo; + } + + public String getProvince() { + return province; + } + + public void setProvince(String province) { + this.province = province; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getBuildingName() { + return buildingName; + } + + public void setBuildingName(String buildingName) { + this.buildingName = buildingName; + } + + public String getBuildingNoName() { + return buildingNoName; + } + + public void setBuildingNoName(String buildingNoName) { + this.buildingNoName = buildingNoName; + } + + public String getPropertyTypeCode() { + return propertyTypeCode; + } + + public void setPropertyTypeCode(String propertyTypeCode) { + this.propertyTypeCode = propertyTypeCode; + } + + public Integer getIsElevator() { + return isElevator; + } + + public void setIsElevator(Integer isElevator) { + this.isElevator = isElevator; + } + + public Integer getAboveFloorAmount() { + return aboveFloorAmount; + } + + public void setAboveFloorAmount(Integer aboveFloorAmount) { + this.aboveFloorAmount = aboveFloorAmount; + } + + public Integer getFloorNumber() { + return floorNumber; + } + + public void setFloorNumber(Integer floorNumber) { + this.floorNumber = floorNumber; + } + + public Integer getDecrCondCode() { + return decrCondCode; + } + + public void setDecrCondCode(Integer decrCondCode) { + this.decrCondCode = decrCondCode; + } + + public Double getStrctureArea() { + return strctureArea; + } + + public void setStrctureArea(Double strctureArea) { + this.strctureArea = strctureArea; + } + + public Integer getTowardDirectCode() { + return towardDirectCode; + } + + public void setTowardDirectCode(Integer towardDirectCode) { + this.towardDirectCode = towardDirectCode; + } + + public Date getUpdateDate() { + return updateDate; + } + + public void setUpdateDate(Date updateDate) { + this.updateDate = updateDate; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/ws/CzbankServiceImpl.java b/projects/fangjia-open/src/main/java/com/fangjia/open/ws/CzbankServiceImpl.java index 06a633b1c..c665678ca 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/ws/CzbankServiceImpl.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/ws/CzbankServiceImpl.java @@ -405,11 +405,7 @@ public class CzbankServiceImpl implements CzbankService { } evaluation.setFacilities(convertToString(fs)); } - -// if (StringUtils.isNotBlank(model.getInsidepos())) { -// evaluation.setInsidepos(model.getInsidepos()); -// } - + evaluation.setCreationDate(new Date()); evaluation.setModificationDate(new Date()); From 9017dd4c2a38ccd5ab68bff046a480a35c6ee274 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 3 Aug 2021 20:06:38 +0800 Subject: [PATCH 006/111] update --- .../BigdataCommunityBatch.groovy | 34 +- .../ccbevaluation/BigdataDealBatch.groovy | 74 ++-- .../ccbevaluation/BigdataForsaleBatch.groovy | 119 +++++-- .../ccbevaluation/CcbCommonUtils.groovy | 62 +++- .../ccbevaluation/StagingDistrictBatch.groovy | 8 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 8 +- .../script/commData/CommonDatautils.groovy | 22 ++ .../script/gj/AddThirdPriceDistrict.groovy | 138 ++++++-- .../fangjia/script/gj/EsSearchUtils.groovy | 33 +- .../fangjia/script/gj/GaoDeMapCrawl.groovy | 10 +- .../com/fangjia/script/gj/RegionPoint.groovy | 11 +- .../fangjia/script/gj/ccb/FileUtils.groovy | 6 +- .../gj/data/BestDistrictPriceExport.groovy | 14 +- .../script/gj/district/DataProcess.groovy | 17 +- .../script/gj/district/ExportData.groovy | 8 +- .../gj/district/MapDistrictProcess.groovy | 153 ++++++-- .../script/gj/district/NameProcess.groovy | 24 +- .../guangfa/GuangfaBlankSelectDistrict.groovy | 335 ++++++++++++++++++ .../script/gj/location/LocationTest.groovy | 79 ++++- .../com/fangjia/script/util/DataUtil.groovy | 4 +- 20 files changed, 931 insertions(+), 228 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 3fa5a820e..7904da09f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.GeoUtils import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -40,12 +41,11 @@ class BigdataCommunityBatch extends BaseBatch{ "总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", "乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) - def rets = createCityDatas(cityCodeDatas, distCodeMap, dists) + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) + def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rows.addAll(rets) println city +" "+rets.size() @@ -57,10 +57,10 @@ class BigdataCommunityBatch extends BaseBatch{ allDatas.addAll(rows) FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false); println "txt 生成" - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") + // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") } - def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){ + def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){ def rets = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -70,7 +70,7 @@ class BigdataCommunityBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - def ret = createData(cityCodeDatas, distCodeMap, dist) + def ret = createData(cityCodeDataMap, distCodeMap, dist) if(ret){ rets << ret } @@ -87,10 +87,10 @@ class BigdataCommunityBatch extends BaseBatch{ return rets; } - def createData(def cityCodeDatas, def distCodeMap, def dist){ + def createData(def cityCodeDataMap, def distCodeMap, def dist){ def ret = [] try{ - ret << cityCodeDatas.city_code + ret << cityCodeDataMap.get(dist.city) if(dist.region){ ret << distCodeMap.get(dist.region) }else{ @@ -101,9 +101,17 @@ class BigdataCommunityBatch extends BaseBatch{ ret << dist.region ret << dist.name ret << dist.address - if(dist.lat){ - ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6) - ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6) + if(dist.lat2){ + def lat = dist.lat2/10000000000d + def lng = dist.lng2/10000000000d + def loactionMap = GeoUtils.bd09ToGps84(lat, lng) + if(loactionMap){ + ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) + ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) + }else{ + ret << "" + ret << "" + } }else{ ret << "" ret << "" @@ -120,7 +128,7 @@ class BigdataCommunityBatch extends BaseBatch{ ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") ret << dist.status ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") - ret << "物业小区" + ret << "pa100" ret << "" if(dist.address){ if(dist.address.toString().endsWith("弄")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 50a1d68b3..641c643b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -27,8 +27,8 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -44,35 +44,43 @@ class BigdataDealBatch extends BaseBatch{ def citys = CcbCommonUtils.citys; def rows = [] def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def trades = getCityData(city) -// def rets = createData(trades, cityCodeDatas, distCodeMap, "other") -// rows.addAll(rets) + def trades = getCityData(city) + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) - def youDataTrade = getYouDaData() - def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda") - rows.addAll(youdaDatas) + println city +" 成交共:"+rets.size() + if(city=="上海"){ + def youDataTrade = getYouDaData() + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } - println city +" "+youdaDatas.size() - FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); + +// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); def cityDatas = [] cityDatas << headers - cityDatas.addAll(rets) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") + cityDatas.addAll(rows) + // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") } - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false); + def fileName = "/home/goojia/ccb/bigdata_deal.txt" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成" - def rets = [] - rets << headers - rets.addAll(rows) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据") +// def rets = [] +// rets << headers +// rets.addAll(rows) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据") } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -114,9 +122,9 @@ class BigdataDealBatch extends BaseBatch{ if(!obj){ return ret; } - ret << cityCodeDatas.city_code + ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() - ret << cityCodeDatas.fangjia_city + ret << trade.city if(trade.fjRegion){ ret << distCodeMap.get(trade.fjRegion) }else{ @@ -215,11 +223,11 @@ class BigdataDealBatch extends BaseBatch{ ret << CcbCodeUtils.getFitmentCode(trade.fitment) ret << "" ret << trade.price - ret << trade.totalPrice + ret << trade.totalPrice * 10000 ret << "" ret << "" ret << "" - ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") ret << "" ret << "" ret << "" @@ -240,7 +248,7 @@ class BigdataDealBatch extends BaseBatch{ ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") ret << "FJW" - ret << trade.totalPrice + ret << trade.totalPrice * 10000 ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") return ret; } @@ -248,9 +256,9 @@ class BigdataDealBatch extends BaseBatch{ def converData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] - ret << cityCodeDatas.city_code + ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() - ret << cityCodeDatas.fangjia_city + ret << trade.city if(trade.region){ ret << distCodeMap.get(trade.region) }else{ @@ -317,7 +325,7 @@ class BigdataDealBatch extends BaseBatch{ ret << "" ret << "" ret << "" - ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd") ret << "" ret << "" ret << "" @@ -373,15 +381,23 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] + def startDate = null; + if(citys.contains(city)){ + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) + }else{ + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + } + + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i0){ BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); for(def s : alias){ @@ -358,8 +392,9 @@ class BigdataForsaleBatch extends BaseBatch{ if(seawater.p_date instanceof String){ seawater.p_date = seawater.p_date.toInteger() } - if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ - rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") rowMap.c_date = rowMap.p_date rowMap.m_date = rowMap.p_date }else{ @@ -518,4 +553,14 @@ class BigdataForsaleBatch extends BaseBatch{ println ids.size() } + def getDistrictIds(def cityid){ + def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'"; + + def distIds = new HashSet() + sqlInstance.eachRow(sql,{it-> + distIds << it.commid + }); + return distIds; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index c11a06cfa..5776037e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -7,7 +7,7 @@ import com.mongodb.DBCollection class CcbCommonUtils { - public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] + public static def citys = ["成都","沈阳","大连","北京", "上海","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","重庆","南京","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -23,21 +23,10 @@ class CcbCommonUtils { public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] - public static def findCityCodeValue(city){ - def q = new BasicDBObject() - q.put("fangjia_city", city) - def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) - def count = dbcursor.count() - if(count >1){ - def data = null - for(def db : dbcursor){ - if(db.city_code){ - data = db - break - } - } - return data; - } + + public static def findCityCodeValue(){ + def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] + return cityCodeMap; } public static def findAllDistrictCodeValue(city){ @@ -105,4 +94,45 @@ class CcbCommonUtils { return lists } + + public static def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + + if(count && avg_price){ + def retMap = [:] + retMap.count = count + retMap.avg_price = avg_price + return retMap + } + return null + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 4ecac813a..8084e319a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{ results << headers results.addAll(rows) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据") - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8"); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\staging_district.xlsx", results, "数据") + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\staging_district.txt",headers, "utf-8", false); } @@ -76,9 +76,9 @@ class StagingDistrictBatch extends BaseBatch{ def getFenQiMap(def dists){ def fenQiMap = [:] for(def d : dists){ - if(d.master_district_name){ +// if(d.master_district_name){ fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d) - } +// } } return fenQiMap; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index d4e252dd5..920ec307a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -19,7 +19,7 @@ import com.mongodb.DBCollection class ThirdPartyPriceBatch extends BaseBatch { def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -30,7 +30,7 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]//CcbCommonUtils.citys; def rows = [] def index = 0 for(def city : citys){ @@ -166,7 +166,9 @@ class ThirdPartyPriceBatch extends BaseBatch { def dbcursor = AVERAGE_STATISTICS.find(q) def map = [:] for(def db : dbcursor){ - map.put(db.region+"##"+db.name, db) + if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ + map.put(db.region+"##"+db.name, db) + } } return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 4c2f2df46..3200b95c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -698,6 +698,28 @@ class CommonDatautils { return rets; } + public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){ + + DBObject q = new BasicDBObject(); + q.put("visible",0) + q.put("cat", cat); + q.put("city", city); + q.put("region", new BasicDBObject('$in', regions)) + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def rets = [] + for(def d : dbsour){ + rets << d + } + return rets; + } + public static def findAllDistrictByQuery(def q, def cat, def fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy index d61bfb2cb..25e05d428 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy @@ -2,6 +2,8 @@ package com.fangjia.script.gj import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.NameProcess import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil @@ -63,7 +65,7 @@ class AddThirdPriceDistrict { def resAddrMap = [:] def filterEstateTypeMap = [:] public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -82,14 +84,77 @@ class AddThirdPriceDistrict { // println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") } def run(){ - exportDatas(); -// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); -// addInfo(); -// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx") -// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx") +// exportDatas(); +// getDataType("D:\\数据\\小区\\南昌第三方均价.xlsx"); +// addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx"); +// addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") +// addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") + dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx") + addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") // addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); } + def dataFilter(filePath){ + ExcelUtils.multiThreadingModify(filePath, { row -> + def name = row['C'] + if(name){ + def flag = NameProcess.filterName(name) + if(!flag){ + row['T'] = 1 + }else{ + if(name.toString().length()<3){ + if(row['N'] && row['N']>=10 && !row['J']){ + row['I'] = 25 + }else{ + row['I'] = 18 + } + + }else{ + def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$")) + if(s1){ + row['I'] = 19 + } + } + } + }else{ + if(!name){ + row['I'] = 1 + } + } + + }); + println "3.名称过滤完成" + } + + def addNewMapInfo(def path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['I']){ + + def gdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "高德") + if(gdData){ + row['F'] = gdData.district + row['G'] = gdData.name + row['H'] = gdData.address + row['I'] = gdData.type + row['J'] = gdData.score + row['K'] = gdData.lng + row['L'] = gdData.lat + } + + def bdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "百度") + if(bdData){ + row['M'] = bdData.district + row['N'] = bdData.name + row['O'] = bdData.address + row['P'] = bdData.type + row['Q'] = bdData.score + row['R'] = bdData.lng + row['S'] = bdData.lat + } + } + },0); + } + def addCrawlMapInfo( def path){ ExcelUtils.multiThreadingModify(path, { row -> def flag = row['R'] @@ -105,27 +170,27 @@ class AddThirdPriceDistrict { row['K'] = data.lat } - def baiduRets = findBaiduMap(row['A'], row['B'], row['C']) - def bdRegion =null - def bdName = null - def bdAddress = null - def bdLng = null - def bdLat = null - def bdEstateType = null - if(baiduRets){ - bdRegion = baiduRets.region - bdName = baiduRets.name - bdAddress = baiduRets.address - bdLng = baiduRets.lng - bdLat = baiduRets.lat - bdEstateType = baiduRets.estateType - } - row['L'] = bdRegion - row['M'] = bdName - row['N'] = bdAddress - row['O'] = bdEstateType - row['P'] = bdLng - row['Q'] = bdLat +// def baiduRets = findBaiduMap(row['A'], row['B'], row['C']) +// def bdRegion =null +// def bdName = null +// def bdAddress = null +// def bdLng = null +// def bdLat = null +// def bdEstateType = null +// if(baiduRets){ +// bdRegion = baiduRets.region +// bdName = baiduRets.name +// bdAddress = baiduRets.address +// bdLng = baiduRets.lng +// bdLat = baiduRets.lat +// bdEstateType = baiduRets.estateType +// } +// row['L'] = bdRegion +// row['M'] = bdName +// row['N'] = bdAddress +// row['O'] = bdEstateType +// row['P'] = bdLng +// row['Q'] = bdLat } },0); @@ -502,8 +567,8 @@ class AddThirdPriceDistrict { return thirdPartyColl.findOne(q) } - def addInfo(){ - + def addInfo(filePath){ + province = "江西" // def city = "苏州" // def datas = findMatchErrorDatas(city); // println "共【"+datas.size()+"】条" @@ -512,7 +577,7 @@ class AddThirdPriceDistrict { // def resMap = groupByName(datas); def index = 0; - ExcelUtils.multiThreadingModify("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx", { row -> + ExcelUtils.multiThreadingModify(filePath, { row -> def flag = row['R'] def dgDatas = row['G'] // if(!flag){ @@ -956,11 +1021,11 @@ class AddThirdPriceDistrict { def queryPoiMap(def city, def region, def name){ final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1") + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1") sqlBuffer.append(" and cityname = '" + city + "'") sqlBuffer.append(" and adname = '" + region + "'") - sqlBuffer.append(" and name = '" + name + "'") - sqlBuffer.append(" and source = '高德'") + sqlBuffer.append(" and MATCH(name,'${name}')") + sqlBuffer.append(" and source = '高德' order by score() desc limit 1") ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) def map = [:] while (rs.next()) { @@ -969,8 +1034,11 @@ class AddThirdPriceDistrict { map.name = rs.getString("name") map.address = rs.getString("address") map.type = rs.getString("type") - map.lng = rs.getString("location.lon") - map.lat = rs.getString("location.lat") + def location = rs.getObject("location") + if(location){ + map.lng = location.x + map.lat = location.y + } } return map; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index 4d50b3661..1d637247d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -64,24 +64,29 @@ class EsSearchUtils { sqlBuffer.append(" and region = '"+region+"'") sqlBuffer.append(" and district_name = '"+name+"'") sqlBuffer.append(" and cat = 'sell'") - sqlBuffer.append(" limit 100") + sqlBuffer.append(" limit 200") ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) def rets = [] while (rs.next()){ - def height = rs.getInt("height") - def direction = rs.getString("direction") - def fitment = rs.getString("fitment") def room = rs.getInt("room") - def hall = rs.getInt("hall") - def toilet = rs.getInt("toilet") - def map = [:] - map.height = height - map.direction = direction - map.fitment = fitment - map.room = room - map.hall = hall - map.toilet = toilet - rets << map + if(room){ + def height = rs.getInt("height") + def direction = rs.getString("direction") + def fitment = rs.getString("fitment") + def hall = rs.getInt("hall") + def toilet = rs.getInt("toilet") + def area = rs.getInt("area") + def map = [:] + map.height = height + map.direction = direction + map.fitment = fitment + map.room = room + map.hall = hall + map.toilet = toilet + map.area = area + rets << map + } + } return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index a8b29aff0..5c5cf081d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -50,14 +50,14 @@ class GaoDeMapCrawl extends BaseBatch{ // def categorys = ["141202", "141203"]//中学,小学 -// def prefix = "district_" -// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 + def prefix = "district_" + def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 // def prefix = "road_" // def categorys = ["190301"]//道路相关 - def prefix = "village_" - def categorys = ["190108","190109"]//村庄 +// def prefix = "village_" +// def categorys = ["190108","190109"]//村庄 // def prefix = "company_" // def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] @@ -119,7 +119,7 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] - def citys =["无锡"] + def citys =["南昌"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy index 17c422651..62d5a00dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -28,8 +28,8 @@ class RegionPointCrawl { static main(args) { def task = new RegionPointCrawl(); //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 -// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"] - def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] +// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] + def provinces = ["江西省"] for(def p : provinces){ task.crawlCityAndDistrictCode(p); task.run(p); @@ -72,7 +72,7 @@ class RegionPointCrawl { for(def cityMap : cityList){ if(cityMap.level == "city"){ saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level); - saveGaoDeRegionCode(province, cityMap.districts); + saveGaoDeRegionCode(province, cityMap.name, cityMap.districts); } } } @@ -81,10 +81,10 @@ class RegionPointCrawl { } } - def saveGaoDeRegionCode(province, districts){ + def saveGaoDeRegionCode(province, cityName,districts){ if(districts){ for(def dist : districts){ - if(dist.level=="district"){ + if(dist.level=="district" || cityName=="东莞市"){ saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); } } @@ -221,4 +221,3 @@ class RegionPointCrawl { return KeyList.get(count); } } -H \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 9382cb398..e7232a3c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -12,8 +12,10 @@ class FileUtils { OutputStream out = new FileOutputStream(fileName, isAppend); def sb = new StringBuffer(); - sb.append(headers.join("|@|")) - sb.append(System.getProperty("line.separator")); + if(headers){ + sb.append(headers.join("|@|")) + sb.append(System.getProperty("line.separator")); + } out.write(sb.toString().getBytes(charset)); sb.setLength(0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy index af92f0489..607d63378 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy @@ -15,7 +15,19 @@ class BestDistrictPriceExport { static main(args) { def task = new BestDistrictPriceExport(); - task.run() + //task.run() + task.run2() + } + + def run2(){ + def filePath = "d:\\数据\\广发\\楼盘.xlsx" + ExcelUtils.multiThreadingModify(filePath, { row -> + def stat = findDistrictPrice(row['A'], row['B'], row['D'], "20210801") + if(stat){ + row['S'] = stat.avg_price + } + println row['A']+" "+row['B']+" "+ row['C']+" "+row['S'] + }) } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy index 7903727f0..87efe314a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy @@ -24,10 +24,10 @@ class DataProcess { static main(args) { //建行信用卡城市(临汾--晋城).xlsx - def filePath = "D:\\数据\\建行银行\\建行信用卡入库小区V1.xlsx" + def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx" def task = new DataProcess(); // task.batchSaveDataToDB(filePath) -// task.mapBatchSaveDataToDB(filePath) + task.mapBatchSaveDataToDB(filePath) // task.calcutePrice(filePath); // task.calcutePrice2(); // task.deleteRepDist("株洲"); @@ -35,7 +35,7 @@ class DataProcess { // task.standartName(filePath); - task.saveGaoDeMap(filePath); +// task.saveGaoDeMap(filePath); // task.updateDBAddress(); // task.updateRegion() @@ -120,13 +120,10 @@ class DataProcess { def lastDatas = [] if(datas){ def keys = [] - for(def data : datas){ - if(data['A']=="厦门" && data['I'] && data['I'].toString().startsWith("24")){ + for(def data : datas){//20,21,25, + if(data['I'] && [26,27].contains(data['I'].toString().toDouble().toInteger())){ def key = data['A']+"##"+data['B']+"##"+data['H'] - if(!keys.contains(key) && data['H'].size()>2){ -// if(data['I'].toInteger()==22){//&& data['N']<3 -// continue; -// } + if(!keys.contains(key) && data['H'] && data['H'].size()>2){ keys << key lastDatas << data } @@ -168,7 +165,7 @@ class DataProcess { addIndex++ def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null) println "${addIndex}【新增入库】"+ values - // FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4); + FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4); }else{ println "【已存在】 ${d['A']} ${d['B']} ${d['H']}" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index 6f9219576..7fa85152f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.elasticsearch.xpack.sql.jdbc.EsDataSource @@ -26,8 +27,9 @@ class ExportData { static main(args) { def task = new ExportData(); -// task.exportThird() - task.exportMap(); + task.exportThird() +// task.exportMap(); + } def exportMap(){ @@ -72,7 +74,7 @@ class ExportData { def exportThird(){ - def citys = ["重庆"] + def citys = ["南昌"] for(def city : citys){ def rets = [] rets << ["城市","区域","名称","别名"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy index f53276b72..56fa2adc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.district +import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DataUtil import com.fangjia.script.commData.CommonDatautils @@ -7,6 +8,7 @@ import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation import java.util.regex.Pattern @@ -15,32 +17,39 @@ class MapDistrictProcess { static main(args) { def task = new MapDistrictProcess(); task.run() +// task.updateRegion() } /** * 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路 - * 20:不同区域新增(库中无) 21:分期新增 22:分数在0.6以上 + * 20:不同区域新增(库中无) 21:分期新增 23:不同区域,距离相近 25:无匹配,可新增 22:有匹配,距离1000米以下,相似度在0.6以上 24:疑似匹配有误 27:人工判定入库 * @return */ def run(){ - def files = ["武汉.xlsx"] +// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"] + def files = ["东莞.xlsx"] for(def fileName : files){ def filePath = "D:\\数据\\小区\\地图\\"+fileName; -// typeFilter(filePath) -// nameStandard(filePath); -// dataFilter(filePath) - matchDistDatas(filePath) + typeFilter(filePath) + nameStandard(filePath); + dataFilter(filePath) houseCalculate(filePath); ////// //统计是否存在多个区域 + matchDistDatas(filePath) moreRegionCalculate(filePath) + addFlag(filePath) mapMatch(filePath); baiduTypeFilter(filePath); - baiduMap(filePath); +// baiduMap(filePath); + distanceFilter(filePath) + predictType(filePath) + noMatchFlag(filePath) + matchFlag(filePath) addFeature(filePath) houseMatch(filePath); - predictType(filePath) + // nameScore(filePath); } @@ -55,7 +64,12 @@ class MapDistrictProcess { row['I'] = 1 }else{ if(name.toString().length()<3){ - row['I'] = 18 + if(row['N'] && row['N']>=10 && !row['J']){ + row['I'] = 25 + }else{ + row['I'] = 18 + } + }else{ def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$")) if(s1){ @@ -70,7 +84,7 @@ class MapDistrictProcess { } }); - println "2.名称过滤完成" + println "3.名称过滤完成" } def typeFilter(filePath){ @@ -84,15 +98,30 @@ class MapDistrictProcess { println "1.类型过滤完成" } + + def baiduTypeFilter(filePath){ def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"] ExcelUtils.multiThreadingModify(filePath, { row -> def type = row['S'] - if(type && !types.contains(type)){ + if(type && row['Q'] && row['Q']==row['H'] && !types.contains(type)){ row['I'] = 1 } }); - println "1.类型过滤完成" + println "9.百度类型过滤完成" + } + + def distanceFilter(filePath){ + ExcelUtils.multiThreadingModify(filePath, { row -> + if(!row['I']){ + if(row['X'] && row['Y']){ + if(row['X']<=500 && row['Y']>=0.5){ + row['I'] = 12 + } + } + } + }); + println "10.距离过滤完成" } def nameStandard(filePath){ @@ -105,12 +134,12 @@ class MapDistrictProcess { } }, 0); - println "1.名称标准化完成" + println "2.名称标准化完成" } def matchDistDatas(def path){ def index = 0 - ExcelUtils.modify(path,{ List row -> + ExcelUtils.multiThreadingModify(path,{ List row -> index ++ if(!row['I'] && !row['J']){ def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']); @@ -126,13 +155,16 @@ class MapDistrictProcess { def maplat = baidu.y row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng) def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L']) + if(score<0.3){ + score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['M']) + } row['Y'] = score } } println index +" "+row } },0) - println "3.小区匹配完成" + println "5.小区匹配完成" } def houseCalculate(filePath){ @@ -141,6 +173,7 @@ class MapDistrictProcess { row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H']) } }); + println "4.房源统计完成" } def moreRegionCalculate(filePath){ @@ -161,8 +194,12 @@ class MapDistrictProcess { } } if(!regions.contains(row['B'])){ - row['O'] = regions.toString() - row['I'] = 20 + if(row['X'] && row['X']<=500 && row['Y'] && row['Y']>=0.5){ + row['I'] = 23 + }else{ + row['O'] = regions.toString() + row['I'] = 20 + } }else if(regions.contains(row['B'])){ row['O'] = "已存在" row['I'] = 10 @@ -170,6 +207,7 @@ class MapDistrictProcess { } } }); + println "6.区域过滤完成" } def addFlag(def filePath){ @@ -206,6 +244,8 @@ class MapDistrictProcess { println index + " " + row } }); + + println "7.打标记完成" } def baiduMap(def filePath){ @@ -233,6 +273,7 @@ class MapDistrictProcess { } } }); + println "11.特征过滤完成" } def mapMatch(def filePath){ @@ -260,16 +301,22 @@ class MapDistrictProcess { println index +" "+row },0); - println "4.地图匹配完成" + println "8.地图匹配完成" } def houseMatch(filePath){ ExcelUtils.multiThreadingModify(filePath, { row -> if(!row['I']) { - def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N']) - if(rMap){ - row['I'] = rMap.flag - } +// def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N']) +// if(rMap){ +// row['I'] = rMap.flag +// } + + if(row['J']){ + if(row['X'] && row['Y'] ){ + + } + } } },0); } @@ -314,6 +361,41 @@ class MapDistrictProcess { },0); } + def noMatchFlag(filePath) { + ExcelUtils.multiThreadingModify(filePath, { row -> + if (!row['I'] && !row['J']) { + if(row['N']>0){ + row['I'] = 25 + }else{ + if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){ + row['I'] = 25 + } + } + + } + }); + } + + def matchFlag(filePath) { + ExcelUtils.multiThreadingModify(filePath, { row -> + if (!row['I'] && row['J'] && row['X'] && row['Y']) { + if(row['X']<=1000 && row['Y']>=0.6){ + row['I'] = 22 + }else if(row['X']<=600 && row['Y']>=0.5){ + row['I'] = 22 + }else if(row['X']<=1300 && row['Y']>=0.8){ + row['I'] = 22 + }else{ + row['I'] = 24 + } + }else if(!row['I'] && row['J'] && row['Y']==0){ + if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){ + row['I'] = 26 + } + } + }); + } + def nameScore(filePath){ ExcelUtils.multiThreadingModify(filePath, { row -> if(!row['I']) { @@ -343,4 +425,31 @@ class MapDistrictProcess { } },0); } + + + def updateRegion(){ + def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx"; + ExcelUtils.multiThreadingModify(filePath, { row -> + def region = getRegionByLocation(row['F'], row['G']); + if(region){ + row['B'] = RegionData.formatRegion(row['A'], region) + } + + },0) + + } + + def getRegionByLocation(lng, lat){ + def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString()) + + if(jsonObject && jsonObject.info=="OK"){ + def compent = jsonObject.regeocode.addressComponent + if(compent){ + def township = compent.township + return township; + } + } + return null + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index f2e4c5d75..92192ed4f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -7,7 +7,9 @@ import java.util.regex.Pattern class NameProcess { static def filterName(def name){ - def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心","养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会"] + def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心", + "养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房", + "合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼"] def isAdd = true; for(def key : keys){ if(name.contains(key)){ @@ -17,7 +19,7 @@ class NameProcess { } if(isAdd){ - def endKeys = ["路","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍"] + def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门"] for(def key : endKeys){ if(name.toString().endsWith(key)){ isAdd = false; @@ -27,7 +29,7 @@ class NameProcess { } if(isAdd){ - def startKeys = ["FQ"] + def startKeys = ["FQ","社区"] for(def key : startKeys){ if(name.toString().endsWith(key)||name.toString().startsWith(key)){ isAdd = false; @@ -36,7 +38,7 @@ class NameProcess { } } if(isAdd){ - def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房"] + def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房", "家属宿舍楼","卫生生活区"] for(def str : strs){ if(str == name){ isAdd = false; @@ -56,7 +58,7 @@ class NameProcess { } } //.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号") - name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim() + name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim() name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","") if(name && name.contains("区")){ def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区")) @@ -191,12 +193,12 @@ class NameProcess { if(str == name){ return null } - if(name){ - s1 = DataUtil.match(name, Pattern.compile("^\\w+")) - if(s1){ - return null - } - } +// if(name){ +// s1 = DataUtil.match(name, Pattern.compile("^\\w+")) +// if(s1){ +// return null +// } +// } return name.toUpperCase() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy new file mode 100644 index 000000000..4643a812a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy @@ -0,0 +1,335 @@ +package com.fangjia.script.gj.guangfa + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong + +class GuangfaBlankSelectDistrict { + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + + + public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; + public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new GuangfaBlankSelectDistrict(); + task.run() + } + + + def run(){ + def file = "D:\\数据\\广发\\获取楼盘信息需求(发送版).xlsx" + def cityMap = standardRegion(file) + def priceMap = getZhiDaoPrice(file) + selectDistricts(cityMap, priceMap) + println "写入完成" + } + + def selectDistricts(cityMap, priceMap){ + def rets = [] + rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"] + cityMap.each{city, regions-> + if(["西安","东莞","杭州"].contains(city)){ + def thirdPriceMap = findNearThirdPartyPrice(city, regions) + def retMaps = findCreditHightDistrict2(city, regions, "2021-08-01", priceMap, thirdPriceMap) + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"]) + //基本情况合格的小区 + def distMap = [:] + for(def d : districts){ + if(regions.contains(d.region)){// && d.household_count && d.household_count>=50 + def distKeys = retMaps.get(d.region) + if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ + if(d.finish_building_date){ + def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) + if(year){ + def mod = 2021-year + if(mod <=15){ + d.house_year = mod + }else{ + d.house_year = null + } + }else{ + d.house_year = null + } + + }else{ + d.house_year = null + } + distMap.put(d.city+"##"+d.region+"##"+d.name, d) + } +// if(d.finish_building_date){ +// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) +// if(year){ +// def mod = 2021-year +// if(mod <=15){ +// def distKeys = retMaps.get(d.region) +// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ +// d.house_year = mod +// distMap.put(d.city+"##"+d.region+"##"+d.name, d) +// } +// } +// } +// } + } + } + println city +" 基本情况合格的小区:"+distMap.size() + + for(def r : regions){ + def names = retMaps.get(r) + if(names){ + def results = getRoomTypeAndTradeInfo(names, distMap, priceMap) + rets.addAll(results) + } + } + println "结束" + } + } + + println "开始写入Excel....."+rets.size() + ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据") + } + + def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){ + + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(names.size()) + + for(def i=0;i + index ++ + if(index >= 4){ + if(row['C']){ + city = row['C'] + } + println city+" "+row['D'] + def newRegion = null + if(city && row['D']){ + newRegion = RegionData.formatRegion(city, row['D']) + row['F'] = newRegion + } + if(city && newRegion){ + def regions = cityMap.get(city) + if(!regions){ + regions = [] + cityMap.put(city, regions) + } + if(!regions.contains(newRegion)){ + regions << newRegion + } + } + } + }); + return cityMap + } + + + def findCreditHightDistrict2(def city, def regions, def weekend, def priceMap, def thirdPriceMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")) + q.put("region", new BasicDBObject('$in', regions)) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + if(regions.contains(d.region)){ + def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name) + if(thirdPrice){ + def precent = (thirdPrice-d.avg_price)*100/d.avg_price + if(Math.abs(precent)>=0 && Math.abs(precent)<=10){ + def price = priceMap.get(d.city +"#"+d.region) + if(d.avg_price>price){ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + } + }else{ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + + } + } + return retMaps; + } + + def getZhiDaoPrice(filePath){ + def priceMap = [:] + def city = null + ExcelUtils.modify(filePath, { row -> + if(row['C']){ + city = row['C'] + } + if(city && row['F']){ + def key = city +"#"+row['F'] + if(!priceMap.containsKey(key)){ + priceMap.put(key, row['E']) + } + } + }) + return priceMap + } + + public static def findShTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("fjCity", dist.city) + q.put("fjRegion", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("fjName", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("fjName", dist.name) + } + + q.put("contractTime", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = youda_esfhouse_coll.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.price){ + total_price = total_price + d.price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + + if(count && avg_price){ + def retMap = [:] + retMap.count = count + retMap.avg_price = avg_price + return retMap + } + return null + } + + def findNearThirdPartyPrice(def city, regions){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", new BasicDBObject('$in', regions)) + q.put("s_date", new BasicDBObject('$gte', 202105)) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def thirdPriceMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.region+"#"+d.name + if(!thirdPriceMap.containsKey(key)){ + thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price) + } + } + return thirdPriceMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index e75f561b2..42d0b1ecc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -24,21 +24,22 @@ class LocationTest { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static main(args) { - Date date=new Date(1622802609); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - String str = format.format(date); - System.out.println(str); +// Date date=new Date(1622802609); +// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); +// String str = format.format(date); +// System.out.println(str); -// def task = new LocationTest() + def task = new LocationTest() // task.run() // task.run2() -// task.run3() + task.run3() // task.run4() } def run(){ - def regionPointsMap = findRegionRange() - def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]); + def city = "南昌" + def regionPointsMap = findRegionRange(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); def index = 0 def rets = [] for(def dist : districts){ @@ -74,7 +75,7 @@ class LocationTest { } } - ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内") + ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内") } def run2(){ @@ -133,10 +134,10 @@ class LocationTest { def run3(){ - ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d-> + ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d-> def distance = null - if(d['G']){ - distance = d['G'].toDouble() + if(d['H']){ + distance = d['H'].toDouble() } if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){ def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期") @@ -180,10 +181,56 @@ class LocationTest { udata.put("m_date", new Date()) DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - def dbo = DbUtils.seaweed.findOne(q) d['K'] = "修正" + } + }else if(d['E'] && d['B'] != d['E'] && distance && distance<=1000){ + def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期") + if(mapName.contains("-") && mapName.endsWith("楼")){ + mapName = mapName.substring(0, mapName.indexOf("-")) + } + if(d['C']==mapName){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("region", d['E']) + q.put("name", d['C']) + q.put("cat", 'district') + q.put("status", 0) + def dbo = DbUtils.seaweed.findOne(q) + if(!dbo){ + q.put("region", d['B']) + def lng2 = null + def lat2 = null + def lng = null + def lat = null + if(d['D']=="高德"){ + lng = (d['I'].toDouble()*10000000000).longValue() + lat = (d['J'].toDouble()*10000000000).longValue() + def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble()) + if(gmap){ + lng2 = (gmap.get("lng")*10000000000).longValue() + lat2 = (gmap.get("lat")*10000000000).longValue() + } + }else{ + lng2 = (d['I'].toDouble()*10000000000).longValue() + lat2 = (d['J'].toDouble()*10000000000).longValue() + def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble()) + if(gmap){ + lng = (gmap.get("lng")*10000000000).longValue() + lat = (gmap.get("lat")*10000000000).longValue() + } + } + def udata = new BasicDBObject() + udata.put("region", d['E']) + udata.put("lng", lng) + udata.put("lat", lat) + udata.put("lng2", lng2) + udata.put("lat2", lat2) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + d['K'] = "修正区域" + } } } },0) @@ -366,13 +413,13 @@ class LocationTest { return flag; } - def findRegionRange(){ + def findRegionRange(city){ def q = new BasicDBObject() - q.put("city","福州市") + q.put("city",city+"市") def dbcursor = POLYLINECOLL.find(q) def regionMap = [:] for(def d : dbcursor){ - def region = RegionData.formatRegion("福州",d.region) + def region = RegionData.formatRegion(city,d.region) regionMap.put(region, d.polyline) } return regionMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy index 8f3aefb49..b18f43f40 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy @@ -531,8 +531,10 @@ public class DataUtil { if(!(address ==~ /.*(交汇|交口|交叉口)$/)){ address += "交汇" } + }else{ + address = null } - address = address.replaceAll("(交汇|交口|交叉口|交汇处).*","交汇") +// address = address.replaceAll("(交汇|交口|交叉口|交汇处).*","交汇") } return address } From fbfd9481ed94437a9009059e2d0d502c30fc508c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 5 Aug 2021 19:45:53 +0800 Subject: [PATCH 007/111] update --- .../BigdataCommbuildtypeBatch.groovy | 15 +- .../BigdataCommunityBatch.groovy | 29 +- .../ccbevaluation/BigdataDealBatch.groovy | 46 ++- .../ccbevaluation/BigdataDistrictBatch.groovy | 11 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 42 +-- .../ccbevaluation/CcbCommonUtils.groovy | 71 ++++- .../script/ccbevaluation/CityBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 37 ++- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 95 +++++- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 254 +++++++++------ .../script/ccbevaluation/TradeStatLog.groovy | 290 ++++++++++++++++++ .../script/gj/AddThirdPriceDistrict.groovy | 53 +++- .../com/fangjia/script/gj/RegionPoint.groovy | 2 +- .../script/gj/district/DataProcess.groovy | 4 +- .../gj/district/MapDistrictProcess.groovy | 2 +- .../script/gj/district/NameProcess.groovy | 4 +- .../gj/price/NoPriceCalcutePrice.groovy | 44 +++ .../gj/test/CrawlDistrictIdSelect.groovy | 2 +- .../TestCrawlTotalCount.groovy | 4 +- .../yinjihaun/amap/GaoDeApiUtils.groovy | 1 + 21 files changed, 806 insertions(+), 204 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 93864962e..2fd71fe46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -31,11 +31,10 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def citys = CcbCommonUtils.citys; def rets = [] for(def city : citys){ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"]) - def rows = createDatas(cityCodeDatas, distCodeMap, dists) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","build_type", "c_date", "m_date"]) + def rows = createDatas(cityCodeDatas, dists) rets.addAll(rows) println city +" "+rows.size() } @@ -45,7 +44,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } - def createDatas(cityCodeDatas, distCodeMap, dists){ + def createDatas(cityCodeDatas, dists){ def rets = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -55,7 +54,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - def ret = createData(cityCodeDatas, distCodeMap, dist) + def ret = createData(cityCodeDatas, dist) if(ret){ rets << ret } @@ -73,7 +72,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ return rets; } - def createData(cityCodeDatas, distCodeMap, dist){ + def createData(cityCodeDatas, dist){ def ret = [] ret << cityCodeDatas.city_code ret << dist._id.toString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 7904da09f..a260ea191 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -26,7 +26,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -44,12 +44,13 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) + println city +" 小区量:"+dists.size() def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rows.addAll(rets) println city +" "+rets.size() - // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); +// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } def allDatas = [] @@ -90,15 +91,26 @@ class BigdataCommunityBatch extends BaseBatch{ def createData(def cityCodeDataMap, def distCodeMap, def dist){ def ret = [] try{ + def data = distCodeMap.get(dist.region) + if(!data){ + println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域" + return ret; + } + ret << cityCodeDataMap.get(dist.city) - if(dist.region){ - ret << distCodeMap.get(dist.region) + if(data){ + ret << data.district_code }else{ ret << "" } ret << dist._id.toString() - ret << dist.region + if(data){ + ret << data.district + }else{ + ret << "" + } + ret << dist.name ret << dist.address if(dist.lat2){ @@ -141,7 +153,10 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "" } }else{ - def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + def jsonObj = null + if(dist.address){ + jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + } if(jsonObj && jsonObj.searchAddress){ ret << jsonObj.searchAddress.street ret << jsonObj.searchAddress.streetNumber diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 641c643b6..27b702dcc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -59,10 +59,7 @@ class BigdataDealBatch extends BaseBatch{ rows.addAll(youdaDatas) println city +" "+youdaDatas.size() } - - // FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); - def cityDatas = [] cityDatas << headers cityDatas.addAll(rows) @@ -95,11 +92,17 @@ class BigdataDealBatch extends BaseBatch{ try { totalCount.addAndGet(1) if(source == "youda"){ - result << converYouDaData(trade, cityCodeDatas, distCodeMap) - println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString() + def ret = converYouDaData(trade, cityCodeDatas, distCodeMap) + if(ret){ + result << ret + println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString() + } }else{ - result << converData(trade, cityCodeDatas, distCodeMap) - println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString() + def ret = converData(trade, cityCodeDatas, distCodeMap) + if(ret){ + result << ret + println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString() + } } @@ -118,20 +121,23 @@ class BigdataDealBatch extends BaseBatch{ def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] - def obj = CommonDatautils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") + def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; + }else{ + println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() ret << trade.city - if(trade.fjRegion){ - ret << distCodeMap.get(trade.fjRegion) + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district }else{ ret << "" + ret << "" } - ret << trade.region - def street = "" def streetNumber = "" if(trade.address){ @@ -256,15 +262,23 @@ class BigdataDealBatch extends BaseBatch{ def converData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] + def data = distCodeMap.get(trade.fjRegion) + if(!data){ + return ret + }else{ + println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" + } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() ret << trade.city - if(trade.region){ - ret << distCodeMap.get(trade.region) + if(data){ + ret << data.district_code + ret << data.district }else{ ret << "" + ret << "" } - ret << trade.region + if(trade.address){ if(trade.address.toString().endsWith("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) @@ -376,7 +390,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"]) def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index 71d367afe..815336ae3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8",false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.txt",headers, "utf-8",false); } def createDatas(rows){ @@ -73,17 +73,14 @@ class BigdataDistrictBatch extends BaseBatch{ def findDistrictCodeValue(city){ def q = new BasicDBObject() q.put("fangjia_city", city) - q.put("district", new BasicDBObject('$ne',"市辖区")) + // q.put("district", new BasicDBObject('$ne',"市辖区")) def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) def lists = [] for(def db : dbcursor){ - if(db.fangjia_district && db.fangjia_district=="二道"){ - println db.fangjia_district+" "+city - } if(db.district_code){ def m = [:] m.id = db._id.toString() - m.dist_name = db.fangjia_district + m.dist_name = db.district m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") m.city_id = db.city_code m.dist_code = db.district_code diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 8c3df2f89..52e7de415 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -69,15 +69,14 @@ class BigdataForsaleBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ - def distIds = getDistrictIds(cityCodeDataMap.get(city)) - +// def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, distIds,20200701, 20210731); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); } } - def exportSellCSV(index,city, cityCodeDataMap, distCodeMap,districts, distIds,startDate, endDate){ + def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) @@ -92,14 +91,16 @@ class BigdataForsaleBatch extends BaseBatch{ try { totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - if(distIds.contains(commId)){ - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) - if(ret && ret.size()<100){ - ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) - } - if(ret){ + + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) + if(ret && ret.size()<100){ + ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) + } + if(ret){ + def areaData = distCodeMap.get(rMap.region) + if(areaData){ + def regionCode = areaData.district_code def cityCode = cityCodeDataMap.get(city) - def regionCode = distCodeMap.get(rMap.region) def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) @@ -107,7 +108,7 @@ class BigdataForsaleBatch extends BaseBatch{ def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); for(def m : ret){ m.city_code = cityCode - m.region = rMap.region + m.region = areaData.district m.region_code = regionCode m.district_id = commId m.district_name = rMap.name @@ -124,9 +125,13 @@ class BigdataForsaleBatch extends BaseBatch{ m.source = "FJW" retMap.put(m.houseId, converData(m)) } + }else{ + println city +" "+rMap.region+" 没有找到对应的区域" } - println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString() + } + println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString() + } catch (Exception e) { e.printStackTrace() } finally { @@ -156,7 +161,6 @@ class BigdataForsaleBatch extends BaseBatch{ // rets << headers // rets.addAll(allDatas) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据") - } @@ -296,9 +300,9 @@ class BigdataForsaleBatch extends BaseBatch{ if(!seawater.height){ seawater.height = null } - if(seawater.height && seawater.height<1){ - continue; - } +// if(seawater.height && seawater.height<1){ +// continue; +// } if(seawater.height && seawater.height>60){ continue; } @@ -311,7 +315,7 @@ class BigdataForsaleBatch extends BaseBatch{ } if(seawater.room){ - if(seawater.room>=5 || seawater.room<1){ + if(seawater.room>=11 || seawater.room<1){ continue; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 5776037e7..1241ab34d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -1,18 +1,22 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.datastore.driver.DataStore import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject class CcbCommonUtils { - public static def citys = ["成都","沈阳","大连","北京", "上海","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","重庆","南京","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + public static def citys = ["北京", "上海","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] - public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") @@ -23,6 +27,7 @@ class CcbCommonUtils { public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static DSCollection dataStoreConnection = null; public static def findCityCodeValue(){ def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] @@ -32,11 +37,11 @@ class CcbCommonUtils { public static def findAllDistrictCodeValue(city){ def q = new BasicDBObject() q.put("fangjia_city", city) - q.put("district", new BasicDBObject('$ne',"市辖区")) + //q.put("district", new BasicDBObject('$ne',"市辖区")) def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - map.put(db.fangjia_district, db.district_code) + map.put(db.fangjia_district, db) } return map; } @@ -135,4 +140,62 @@ class CcbCommonUtils { return null } + public static DSCollection getDataStoreConnection(){ + if(dataStoreConnection != null) { + return dataStoreConnection + } + DataStore ds = new DataStore("seaweed.db.fangjia.com", 27007) + dataStoreConnection = ds.getCollection("fangjia.seaweed") + return dataStoreConnection + } + + public static def findAllDistrictByCity(def city, def cat, def fields){ + + DBObject q = new BasicDBObject(); + q.put("visible",0) + q.put("cat", cat); + q.put("city", city); +// q.put("name_cat", 0); +// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def rets = [] + for(def d : dbsour){ + if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){ + rets << d + } + } + return rets; + } + + public static def findDistrict(String city, String region, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + if(region){ + q.put("region", region); + } + + q.put("name", name); + q.put("cat", cat) + q.put("status", 0) + q.put("name_cat", 0); + q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) + + + DSCursor cursor = getDataStoreConnection().find(q); + if(cursor){ + for(def obj : cursor){ + return obj; + } + } + return null; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy index ef52b58c6..db72da8b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -26,7 +26,7 @@ class CityBatch extends BaseBatch{ def rows = [] def index = 1 for(def city : citys){ - def data = CcbCommonUtils.findCityCodeValue(city) + def data = CcbCommonUtils.findCityCodeValue() def map = [:] map.name = city if(data){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 234d172b5..49c60f702 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -31,7 +31,7 @@ class GuiDingPriceBatch extends BaseBatch{ def rows = [] for (def city : citys) { - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) @@ -61,25 +61,24 @@ class GuiDingPriceBatch extends BaseBatch{ def dist = distMap.get(ps.region+"##"+ps.name) if(dist){ def ret = [] - ret << ps._id.toString() - ret << dist._id.toString() - ret << dist.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city - if(dist.region){ - ret << distCodeMap.get(dist.region) - }else{ + def areaData = distCodeMap.get(dist.region) + if(areaData){ + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + ret << areaData.district_code + ret << areaData.district + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << ps.guiding_value + ret << ps.guiding_person ret << "" + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret } - ret << dist.region - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") - ret << ps.guiding_value - ret << ps.guiding_person - ret << "" - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - rets << ret } } } @@ -87,7 +86,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def getDistrictMap(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def districts = CcbCommonUtils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 8084e319a..c73edb18e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -27,7 +27,7 @@ class StagingDistrictBatch extends BaseBatch{ def citys = CcbCommonUtils.citys; def rows = [] for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","master_district_name", "c_date", "m_date"]) def fenQiMap = getFenQiMap(dists); def rets = createCityData(dists, fenQiMap) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 399e3c53a..5f482e0a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1,21 +1,96 @@ package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject import groovy.sql.Sql def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -def citys = CcbCommonUtils.citys; -def rows = [] +//def citys = CcbCommonUtils.citys; +//def rows = [] +// +//for(def city : citys) { +// def data = CcbCommonUtils.findCityCodeValue() +// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +// +// def count = 0 +// sqlInstance.execute(sql,{ +// count = it.count +// }); +// println city +" "+count +//} -for(def city : citys) { - def data = CcbCommonUtils.findCityCodeValue(city) - def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0802(3).xlsx" +//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\成交统计.xlsx",0,1) +//def dealMap = [:] +//for(def d : datas){ +// dealMap.put(d['A'], d['B']) +//} +// +// +//ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['C']){ +// d['P'] = dealMap.get(d['C']) +// } +//},2) - def count = 0 - sqlInstance.execute(sql,{ - count = it.count - }); - println city +" "+count + +def city = "中山" +def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +def index =0 +def updateIndex = 0 +def delIndex = 0 +for(def dist : dists){ + if(dist.region=="石岐区街道"){ + def lat = dist.lat/10000000000.0 + def lng = dist.lng/10000000000.0 + def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) + if(regionStr){ + def region = RegionData.formatRegion(dist.city, regionStr) + def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") + if(obj){ + delIndex++ + DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) + println delIndex+" [del]"+region +" "+dist.name + }else{ + updateIndex++ + DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) + println updateIndex +" [update]"+region+" "+dist.name + } + }else{ + index++ + println index+" [没找到]"+dist.name + } + } +} + +def getStreetInfo(def city, def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent && addressComponent.city.contains(city)){ + township = addressComponent.township + } + } + } + return township; } + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 920ec307a..c4e81d699 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -30,15 +30,15 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def index = 0 for(def city : citys){ index++ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2"]) def thirdPriceMap = findThirdPartyPrice(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) rows.addAll(rets) @@ -55,103 +55,102 @@ class ThirdPartyPriceBatch extends BaseBatch { for(def d : dists){ def third = thirdPriceMap.get(d.region+"##"+d.name) if(third){ - def ret = [] - ret << third._id.toString() - ret << d._id.toString() - ret << d.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city - if(d.region){ - ret << distCodeMap.get(d.region) - }else{ - ret << "" - } - ret << d.region - ret << third.s_date + def data = distCodeMap.get(d.region) + if(data){ + def ret = [] + ret << third._id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + ret << data.district_code + ret << data.district + ret << third.s_date - if(third.zhugezhaofang_esf_price){ - ret << third.zhugezhaofang_esf_price - }else{ - ret << "" - } - if(third.anjuke_esf_price){ - ret << third.anjuke_esf_price - }else{ - ret << "" - } - if(third.soufang_esf_price){ - ret << third.soufang_esf_price - }else{ - ret << "" - } + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } - if(third.lianjia_esf_price){ - ret << third.lianjia_esf_price - }else{ - ret << "" - } + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } - if(third.cityhouse_esf_price){ - ret << third.cityhouse_esf_price - }else{ - ret << "" - } + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } - if(third.maitian_esf_price){ - ret << third.maitian_esf_price - }else{ - ret << "" - } + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } - if(third.beike_esf_price){ - ret << third.beike_esf_price - }else{ - ret << "" - } + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } - if(third.get("365house_esf_price")){ - ret << third.get("365house_esf_price") - }else{ - ret << "" - } + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } - if(third.zhongyuan_esf_price){ - ret << third.zhongyuan_esf_price - }else{ - ret << "" - } + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } - if(third.tuboshi_esf_price){ - ret << third.tuboshi_esf_price - }else{ - ret << "" - } - if(third.qfang_esf_price){ - ret << third.qfang_esf_price - }else{ - ret << "" - } + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } - if(third.fangtan_esf_price){ - ret << third.fangtan_esf_price - }else{ - ret << "" - } + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } - if(third.fangduoduo_esf_price){ - ret << third.fangduoduo_esf_price - }else{ - ret << "" + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret } - if(third.get("5i5j_esf_price")){ - ret << third.get("5i5j_esf_price") - }else{ - ret << "" - } - ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - rets << ret } } @@ -162,15 +161,82 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city){ def q = new BasicDBObject() q.put("city", city) - q.put("s_date", curDate) - def dbcursor = AVERAGE_STATISTICS.find(q) + q.put("s_date", new BasicDBObject('$gte', curDate)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ - if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ - map.put(db.region+"##"+db.name, db) + def key = db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){ + dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf")){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf")) + } + if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){ + dataMap.put("soufang_esf_price", db.soufang_esf_price) + } + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + + if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price")){ + dataMap.put("365house_esf_price", db.get("365house_esf_price")) + } + + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + + if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price){ + dataMap.put("qfang_esf_price", db.qfang_esf_price) + } + + if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price){ + dataMap.put("fangtan_esf_price", db.fangtan_esf_price) + } + + if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price){ + dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price) + } + + if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price")){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } +// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ +// map.put(key, db) +// } + } + + def resultMap = [:] + map.each {key, value-> + if(value){ + resultMap.put(key, value) } } - return map + return resultMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy new file mode 100644 index 000000000..467a3a9f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -0,0 +1,290 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +/*** + * 建行成交抓取统计 + */ +class TradeStatLog { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; + public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new TradeStatLog() + //task.run() + task.thirdPartyStat() + + } + + def run(){ + def citys = CcbCommonUtils.citys; + def rets = [] + rets << ["城市","5月总量","6月总量","贝壳","链家", "我爱我家","房天下","中原"] + def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] + def startDate = "2021-05-01" + def date = "2021-06-01" + def endDate = "2021-07-01" + for(def city : citys){ + def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) + def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + def ret = [] + ret << city + + if(city == "上海"){ + def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) + def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger() + def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger() + ret << a + ret << b + def c = (b-a)*100/a + def pre = DataHandleUtils.getDoubleRound(2, c) + ret << pre + }else{ + ret << lastMonthCount + ret << curMonthCount + + if(lastMonthCount){ + def a = (curMonthCount-lastMonthCount)*100/lastMonthCount + def pre = DataHandleUtils.getDoubleRound(2, a) + ret << pre + }else{ + ret << null + } + + } + + for(def source : sources){ + def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) + def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + ret << sourceLastMonthCount + ret << sourceCurMonthCount + } + rets << ret + println ret + } + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据") + println "表格生成完成" + } + + + def thirdPartyStat(){ + def citys = CcbCommonUtils.citys; + def rets = [] + rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"] + for(def city : citys){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2"]); + def m = getThirdPartyPricePrecent(city) + rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14] + } + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据") + println "表格生成完成" + } + + + public static def findTradesCount(city, source, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + if(source){ + q.put("source", source) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + def getYouDaData(startDate, endDate){ + def q = new BasicDBObject() + q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lt', endDate)); + def count = youda_esfhouse_coll.count(q) + return count; + } + + + def getThirdPartyPricePrecent(def cityid){ + def sql = "SELECT * FROM third_party_price WHERE cityid = '"+cityid+"'"; + + def count0 = 0 + def count1 = 0 + def count2 = 0 + def count3 = 0 + def count4 = 0 + def count5 = 0 + def count6 = 0 + def count7 = 0 + def count8 = 0 + def count9 = 0 + def count10 = 0 + def count11 = 0 + def count12 = 0 + def count13 = 0 + def count14 = 0 + def total = 0 + sqlInstance.eachRow(sql,{third-> + total++ + def ret = [] + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + } + + if(ret.size()==0){ + count0++ + } + if(ret.size()==1){ + count1++ + } + if(ret.size()==2){ + count2++ + } + if(ret.size()==3){ + count3++ + } + if(ret.size()==4){ + count4++ + } + if(ret.size()==5){ + count5++ + } + + if(ret.size()==6){ + count6++ + } + if(ret.size()==7){ + count7++ + } + if(ret.size()==8){ + count8++ + } + if(ret.size()==9){ + count9++ + } + if(ret.size()==10){ + count10++ + } + if(ret.size()==11){ + count11++ + } + if(ret.size()==12){ + count12++ + } + if(ret.size()==13){ + count13++ + } + if(ret.size()==14){ + count14++ + } + }); + def pre1 = 0 + def pre2 = 0 + def pre3 = 0 + def pre4 = 0 + def pre5 = 0 + def pre6 = 0 + def pre7 = 0 + def pre8 = 0 + def pre9 = 0 + def pre10 = 0 + def pre11 = 0 + def pre12 = 0 + def pre13 = 0 + def pre14 = 0 + if(total){ + pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total) + pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total) + pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total) + pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total) + pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total) + pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total) + pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total) + pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total) + pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total) + pre10 = DataHandleUtils.getDoubleRound(2, count10*100.0/total) + pre11 = DataHandleUtils.getDoubleRound(2, count11*100.0/total) + pre12 = DataHandleUtils.getDoubleRound(2, count12*100.0/total) + pre13 = DataHandleUtils.getDoubleRound(2, count13*100.0/total) + pre14 = DataHandleUtils.getDoubleRound(2, count14*100.0/total) + } + def map = [:] + map.total = total + map.count1 = pre1 + map.count2 = pre2 + map.count3 = pre3 + map.count4 = pre4 + map.count5 = pre5 + map.count6 = pre6 + map.count7 = pre7 + map.count8 = pre8 + map.count9 = pre9 + map.count10 = pre10 + map.count11 = pre11 + map.count12 = pre12 + map.count13 = pre13 + map.count14 = pre14 + return map + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy index 25e05d428..579e0710e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy @@ -1,13 +1,16 @@ package com.fangjia.script.gj import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import org.bson.types.ObjectId @@ -89,7 +92,8 @@ class AddThirdPriceDistrict { // addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx"); // addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") // addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") - dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx") + // dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx") +// matchDistDatas("D:\\数据\\小区\\南昌第三方均价.xlsx") addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") // addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); } @@ -104,21 +108,21 @@ class AddThirdPriceDistrict { }else{ if(name.toString().length()<3){ if(row['N'] && row['N']>=10 && !row['J']){ - row['I'] = 25 + row['T'] = 25 }else{ - row['I'] = 18 + row['T'] = 18 } }else{ def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$")) if(s1){ - row['I'] = 19 + row['T'] = 19 } } } }else{ if(!name){ - row['I'] = 1 + row['T'] = 1 } } @@ -578,7 +582,7 @@ class AddThirdPriceDistrict { def index = 0; ExcelUtils.multiThreadingModify(filePath, { row -> - def flag = row['R'] + def flag = row['T'] def dgDatas = row['G'] // if(!flag){ index++ @@ -612,9 +616,9 @@ class AddThirdPriceDistrict { // def estateType = filterEstateTypeMap.get(row['A']+"##"+row['B']+"##"+row['C']) // row['W'] = estateType - if(!row['X']){ - def name = nameProcess(row['B'], row['C']) - row['X'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C']) + if(!row['Z']){ + //def name = nameProcess(row['B'], row['C']) + row['Z'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C']) } // @@ -1053,6 +1057,37 @@ class AddThirdPriceDistrict { return null } + def matchDistDatas(def path){ + def index = 0 + ExcelUtils.multiThreadingModify(path,{ List row -> + index ++ + if(!row['I'] && !row['J']){ + def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['C'], row['D']); + if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){ + //row['J'] = rMap.city + row['U'] = rMap.region + row['V'] = rMap.name + row['W'] = rMap.address + + if(rMap.lat){ +// def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3) +// def maplng = baidu.x +// def maplat = baidu.y +// row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng) + def score = CosineSimilarAlgorithm.getSimilarity(row['C'], row['V']) + if(score<0.3){ + score = CosineSimilarAlgorithm.getSimilarity(row['D'], row['V']) + } + row['X'] = score + } + } + println index +" "+row + } + },0) + println "5.小区匹配完成" + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy index 62d5a00dc..d110c793f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -29,7 +29,7 @@ class RegionPointCrawl { def task = new RegionPointCrawl(); //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] - def provinces = ["江西省"] + def provinces = ["吉林省"] for(def p : provinces){ task.crawlCityAndDistrictCode(p); task.run(p); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy index 87efe314a..51632fe04 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy @@ -24,7 +24,7 @@ class DataProcess { static main(args) { //建行信用卡城市(临汾--晋城).xlsx - def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx" + def filePath = "D:\\数据\\小区\\地图\\张家口.xlsx" def task = new DataProcess(); // task.batchSaveDataToDB(filePath) task.mapBatchSaveDataToDB(filePath) @@ -121,7 +121,7 @@ class DataProcess { if(datas){ def keys = [] for(def data : datas){//20,21,25, - if(data['I'] && [26,27].contains(data['I'].toString().toDouble().toInteger())){ + if(data['I'] && [20,21,25,26,27].contains(data['I'].toString().toDouble().toInteger())){ def key = data['A']+"##"+data['B']+"##"+data['H'] if(!keys.contains(key) && data['H'] && data['H'].size()>2){ keys << key diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy index 56fa2adc2..ca4e29905 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy @@ -28,7 +28,7 @@ class MapDistrictProcess { def run(){ // def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"] - def files = ["东莞.xlsx"] + def files = ["青岛.xlsx","沈阳.xlsx","哈尔滨.xlsx","张家口.xlsx","大连.xlsx"] for(def fileName : files){ def filePath = "D:\\数据\\小区\\地图\\"+fileName; typeFilter(filePath) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index 92192ed4f..3b2c217fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -9,7 +9,7 @@ class NameProcess { static def filterName(def name){ def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心", "养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房", - "合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼"] + "合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼","养生"] def isAdd = true; for(def key : keys){ if(name.contains(key)){ @@ -19,7 +19,7 @@ class NameProcess { } if(isAdd){ - def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门"] + def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门","学校","公司"] for(def key : endKeys){ if(name.toString().endsWith(key)){ isAdd = false; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy new file mode 100644 index 000000000..07bb6b070 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.gj.price + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class NoPriceCalcutePrice { + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + static main(args) { + def task = new NoPriceCalcutePrice() + task.run() + } + def run(){ + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"]) + def map = getDistrictByCity(city, "2021-08-01") + for(def d : districts){ + if(!map.containsKey(d.region+"@"+d.name)){ + FangjiaClientUtils.sendDistrictPriceCalculate(d.city, d.region, d.name, 12); + println d.city+" "+d.region+" "+d.name + } + } + } + } + + def getDistrictByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")) + def dbcursor = DIST_SELL_STAT.find(q) + def map = [:] + for(def d : dbcursor){ + map.put(d.region+"@"+d.district_name, d.avg_price) + } + return map + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index d436aa081..50ffcf9dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -61,7 +61,7 @@ class CrawlDistrictIdSelect { def findCrawlData(){ def q = new BasicDBObject() // q.put("city","福州") - q.put("source", "安居客") +// q.put("source", "安居客") q.put("s_date",202012) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 24b482d1e..f6887e506 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202105); - def count2 = findTotalCount(source, 202106); + def count1 = findTotalCount(source, 202106); + def count2 = findTotalCount(source, 202107); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 781687012..2f146d11f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -26,6 +26,7 @@ class GaoDeApiUtils { Map params = new HashMap() params.put("key", GAO_DE_AK) params.put("location", location) + params.put("extensions","all") String content try { content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") From b77999856be6983927f0e09253271aef6c0b26a5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 5 Aug 2021 20:48:45 +0800 Subject: [PATCH 008/111] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E5=8E=BB=E7=A9=BA?= =?UTF-8?q?=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataCommbuildtypeBatch.groovy | 2 +- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 4 ++++ .../fangjia/script/ccbevaluation/StagingDistrictBatch.groovy | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 2fd71fe46..d8e1a8ece 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -33,7 +33,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ for(def city : citys){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","build_type", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"]) def rows = createDatas(cityCodeDatas, dists) rets.addAll(rows) println city +" "+rows.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 1241ab34d..1a5bc0a41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DSCursor import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -168,6 +169,9 @@ class CcbCommonUtils { def rets = [] for(def d : dbsour){ if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){ + if(d.address){ + d.address = d.address.replaceAll("\r|\n|\\s", ""); + } rets << d } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index c73edb18e..d945175e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -27,7 +27,7 @@ class StagingDistrictBatch extends BaseBatch{ def citys = CcbCommonUtils.citys; def rows = [] for(def city : citys){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","master_district_name", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","master_district_name", "c_date", "m_date"]) def fenQiMap = getFenQiMap(dists); def rets = createCityData(dists, fenQiMap) rows.addAll(rets) From bedab9b301c3e8bc111d5de38f6a49a76cf415c6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 5 Aug 2021 20:53:44 +0800 Subject: [PATCH 009/111] update --- .../com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 27b702dcc..84ac44d09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -390,7 +390,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","build_type","build_structure", "house_property","property_life"]) def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) From 208d52949ef6c65eb851ff8e40ae20337efd9829 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 5 Aug 2021 21:14:38 +0800 Subject: [PATCH 010/111] path update --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index c4e81d699..f4b248b9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -44,8 +44,16 @@ class ThirdPartyPriceBatch extends BaseBatch { rows.addAll(rets) println index+" "+city+" "+rets.size() } + + + + def fileName = "/home/goojia/ccb/third_party_price.txt" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt" + } + def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); } From e423ab16ccbb473688987d1946ef17e137198c13 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 6 Aug 2021 17:38:41 +0800 Subject: [PATCH 011/111] update --- .../BigdataCommbuildtypeBatch.groovy | 10 +++--- .../ccbevaluation/BigdataDealBatch.groovy | 10 +++--- .../ccbevaluation/BigdataForsaleBatch.groovy | 10 +++--- .../ccbevaluation/CcbCommonUtils.groovy | 16 ++++----- .../ccbevaluation/EvaluateRateBatch.groovy | 2 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 9 ++--- .../district/SeaweedDataMatchProcess.groovy | 21 ++++++----- .../script/gj/location/LocationTest.groovy | 6 ++-- .../script/gj/price/EvaluationBatch.groovy | 36 ++++++++++--------- .../fangjia/script/util/OpenApiClient.groovy | 2 +- 10 files changed, 64 insertions(+), 58 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index d8e1a8ece..950a6a697 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -34,7 +34,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"]) - def rows = createDatas(cityCodeDatas, dists) + def rows = createDatas(cityCodeDatas.get(city), dists) rets.addAll(rows) println city +" "+rows.size() } @@ -44,7 +44,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } - def createDatas(cityCodeDatas, dists){ + def createDatas(cityCode, dists){ def rets = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -54,7 +54,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - def ret = createData(cityCodeDatas, dist) + def ret = createData(cityCode, dist) if(ret){ rets << ret } @@ -72,9 +72,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ return rets; } - def createData(cityCodeDatas, dist){ + def createData(cityCode, dist){ def ret = [] - ret << cityCodeDatas.city_code + ret << cityCode ret << dist._id.toString() ret << CcbCodeUtils.parseBuildType(dist.build_type) ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 84ac44d09..37b58553a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -124,8 +124,6 @@ class BigdataDealBatch extends BaseBatch{ def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; - }else{ - println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() @@ -262,11 +260,11 @@ class BigdataDealBatch extends BaseBatch{ def converData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] - def data = distCodeMap.get(trade.fjRegion) + def data = distCodeMap.get(trade.region) if(!data){ return ret }else{ - println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" + println trade.city +" "+trade.region+" 没有找到对应的区域" } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() @@ -399,7 +397,7 @@ class BigdataDealBatch extends BaseBatch{ def startDate = null; if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) }else{ startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) } @@ -415,7 +413,7 @@ class BigdataDealBatch extends BaseBatch{ if(trades){ tradesList.addAll(trades) } - println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString() + println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString() } catch (Exception e) { e.printStackTrace() } finally { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 52e7de415..03c419ca5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -64,7 +64,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["陵水"]//CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ @@ -157,10 +157,10 @@ class BigdataForsaleBatch extends BaseBatch{ FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); println "txt文件生成 " -// def rets = [] -// rets << headers -// rets.addAll(allDatas) -// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据") + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 1a5bc0a41..cff67d153 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -12,7 +12,7 @@ import com.mongodb.DBObject class CcbCommonUtils { - public static def citys = ["北京", "上海","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + public static def citys = ["上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -185,18 +185,18 @@ class CcbCommonUtils { if(region){ q.put("region", region); } - q.put("name", name); q.put("cat", cat) q.put("status", 0) - q.put("name_cat", 0); - q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) - - DSCursor cursor = getDataStoreConnection().find(q); if(cursor){ - for(def obj : cursor){ - return obj; + for(def d : cursor){ + if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){ + if(d.address){ + d.address = d.address.replaceAll("\r|\n|\\s", ""); + } + return d; + } } } return null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index be208aebb..5a0c7a328 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -52,7 +52,7 @@ class EvaluateRateBatch extends BaseBatch{ } private def findEvaluationRate(){ - def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'"; + def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'"; def rets = [] sqlInstance.eachRow(sql,{ rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index f4b248b9c..a81e03ba1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -66,11 +66,11 @@ class ThirdPartyPriceBatch extends BaseBatch { def data = distCodeMap.get(d.region) if(data){ def ret = [] - ret << third._id.toString() + ret << third.id.toString() ret << d._id.toString() ret << d.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city + ret << cityCodeDatas.get(d.city) + ret << d.city ret << data.district_code ret << data.district ret << third.s_date @@ -177,6 +177,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def dataMap = map.get(key) if(!dataMap){ dataMap = [:] + dataMap.put("id", db._id.toString()) map.put(key, dataMap) } if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){ @@ -240,7 +241,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def resultMap = [:] map.each {key, value-> - if(value){ + if(value && value.size()>1){ resultMap.put(key, value) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 5b0910fc8..96b55d4c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -15,7 +15,10 @@ class SeaweedDataMatchProcess { if(!rMap.city){ def obj = CommonDatautils.findDistrict(city, region, name, "district") if(!obj){ - obj = CommonDatautils.findDistrictByAlias(city, region, name, "district") + def datas = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(datas){ + obj = datas[0] + } } if(obj){ rMap.estate_type2 = obj.estate_type2 @@ -40,14 +43,14 @@ class SeaweedDataMatchProcess { - if(rMap && rMap.id && region.contains(rMap.region)) { - def obj = CommonDatautils.findDistrictById(rMap.id.toString()); - if (obj) { - rMap.estate_type2 = obj.estate_type2 - } - }else{ - rMap = null - } +// if(rMap && rMap.id && region.contains(rMap.region)) { +// def obj = CommonDatautils.findDistrictById(rMap.id.toString()); +// if (obj) { +// rMap.estate_type2 = obj.estate_type2 +// } +// }else{ +// rMap = null +// } if(!rMap && name && name.contains("·")){ def newName = trimChar(name).replace("·","") rMap = matchFromDB2(city,newName) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index 42d0b1ecc..3653fd95e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -30,14 +30,14 @@ class LocationTest { // System.out.println(str); def task = new LocationTest() -// task.run() + task.run() // task.run2() - task.run3() +// task.run3() // task.run4() } def run(){ - def city = "南昌" + def city = "杭州" def regionPointsMap = findRegionRange(city) def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy index dca12c36a..40b17049e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.price +import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.ExcelUtils @@ -11,32 +12,35 @@ class EvaluationBatch { task.run() } def run(){ - def filePath = "D:\\数据\\农行\\东莞农行测试数据(御景花园18条估值)(人工询价).xlsx" + def filePath = "D:\\数据\\银联\\房价网测试数据.xlsx" def city = "东莞" ExcelUtils.modify(filePath, { data -> if(!data['M']){ // def address = data['C'].replace("\"","").trim() // def rMap = SeaweedDataMatchProcess.doMatch(city, data['K'],data['L'], address ); - def rMap = SeaweedDataMatchProcess.doMatch(city, data['D'],data['E'], null); - if(rMap){ - def json = evaluation(rMap, data['H'], 7, data['I'],null) + def region = RegionData.formatRegion(data['A'], data['B']) + if(region){ + def rMap = SeaweedDataMatchProcess.doMatch(data['A'], region, data['C'], null); + if(rMap){ + def json = evaluation(rMap, data['D'], null, null,null) -// data['J'] = rMap.city -// data['K'] = rMap.region -// data['L'] = rMap.name - if(json){ - def avgPrice = json.avg_price - def totalPrice = json.total_price - - data['N'] = totalPrice - data['M'] = avgPrice - data['O'] = json.credit - data['P'] = rMap.region - data['Q'] = rMap.name + data['E'] = rMap.city + data['F'] = rMap.region + data['G'] = rMap.name + if(json){ + def avgPrice = json.avg_price + def totalPrice = json.total_price + data['H'] = totalPrice + data['I'] = avgPrice + data['J'] = json.credit +// data['P'] = rMap.region +// data['Q'] = rMap.name + } } } + } }, 0); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 841fb2f08..6296d9ba8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -625,7 +625,7 @@ public class OpenApiClient { params.put("token", TOKEN) try { JSONObject result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/evaluate", params) - if (result.getInt("code") == 200) { + if (result && result.getInt("code") == 200) { if (result.containsKey("result")) { // if (result.get("result").get("credit") == 0) { // return null From 6cde530b4f52808c5d9c6c5dc03e0ceabb68dfbc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 7 Aug 2021 11:39:27 +0800 Subject: [PATCH 012/111] update --- .../script/ccbevaluation/BigdataCommbuildtypeBatch.groovy | 1 + .../fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy | 2 +- .../groovy/com/fangjia/script/gj/location/LocationTest.groovy | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 950a6a697..f36fb42b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -83,5 +83,6 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ ret << "FJW" ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") return ret; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index a260ea191..1100e386f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -44,7 +44,7 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" 小区量:"+dists.size() def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index 3653fd95e..aaeb0cdef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -327,7 +327,7 @@ class LocationTest { region = r.district address = r.address - if(r.district.contains(dist.region)){ + if(r.district && r.district.contains(dist.region)){ score = 10 }else { score = 5 @@ -367,7 +367,7 @@ class LocationTest { region = r.district mapName = r.name address = r.address - if(r.district.contains(dist.region)){ + if(r.district && r.district.contains(dist.region)){ score = 8 }else { score = 5 From 2979ca3b220d6373e770f033c1861d5f41455260 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 7 Aug 2021 12:47:16 +0800 Subject: [PATCH 013/111] update --- .../fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 1100e386f..50c71cf74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -116,7 +116,7 @@ class BigdataCommunityBatch extends BaseBatch{ if(dist.lat2){ def lat = dist.lat2/10000000000d def lng = dist.lng2/10000000000d - def loactionMap = GeoUtils.bd09ToGps84(lat, lng) + def loactionMap = GeoUtils.gcjToGps84(lat, lng) if(loactionMap){ ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) From bb8554421e8c2f9022ea7e5e17e4e2e39a9b8453 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 7 Aug 2021 13:26:57 +0800 Subject: [PATCH 014/111] update --- .../script/ccbevaluation/BigdataCommunityBatch.groovy | 6 +++--- .../main/groovy/com/fangjia/script/gj/RegionPoint.groovy | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 50c71cf74..7aff7834c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -113,9 +113,9 @@ class BigdataCommunityBatch extends BaseBatch{ ret << dist.name ret << dist.address - if(dist.lat2){ - def lat = dist.lat2/10000000000d - def lng = dist.lng2/10000000000d + if(dist.lat){ + def lat = dist.lat/10000000000d + def lng = dist.lng/10000000000d def loactionMap = GeoUtils.gcjToGps84(lat, lng) if(loactionMap){ ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy index d110c793f..4d603f258 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -29,7 +29,7 @@ class RegionPointCrawl { def task = new RegionPointCrawl(); //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] - def provinces = ["吉林省"] + def provinces = ["浙江省"] for(def p : provinces){ task.crawlCityAndDistrictCode(p); task.run(p); From bcc788c1aa09fa796fab2eea7f872a58d10b0edf Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 7 Aug 2021 14:19:34 +0800 Subject: [PATCH 015/111] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 64 ++++++++++--------- .../script/gj/location/LocationTest.groovy | 4 +- .../script/looplineinfo/InsidePolygon.groovy | 22 ++++--- 4 files changed, 49 insertions(+), 43 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 37b58553a..3a252731e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -402,7 +402,7 @@ class BigdataDealBatch extends BaseBatch{ startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) } - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i total++ def ret = [] - if(third.zhugezhaofang_esf_price){ - ret << third.zhugezhaofang_esf_price + if(third.zhuge_avg_price){ + ret << third.zhuge_avg_price } - if(third.anjuke_esf_price){ - ret << third.anjuke_esf_price + if(third.anjuke_avg_price){ + ret << third.anjuke_avg_price } - if(third.soufang_esf_price){ - ret << third.soufang_esf_price + if(third.fang_avg_price){ + ret << third.fang_avg_price } - if(third.lianjia_esf_price){ - ret << third.lianjia_esf_price + if(third.lianjia_avg_price){ + ret << third.lianjia_avg_price } - if(third.cityhouse_esf_price){ - ret << third.cityhouse_esf_price + if(third.cityhouse_avg_price){ + ret << third.cityhouse_avg_price } - if(third.maitian_esf_price){ - ret << third.maitian_esf_price + if(third.maitian_avg_price){ + ret << third.maitian_avg_price } - if(third.beike_esf_price){ - ret << third.beike_esf_price + if(third.ke_avg_price){ + ret << third.ke_avg_price } - if(third.get("365house_esf_price")){ - ret << third.get("365house_esf_price") + if(third.taofang_avg_price){ + ret << third.taofang_avg_price } - if(third.zhongyuan_esf_price){ - ret << third.zhongyuan_esf_price + if(third.zhongyuan_avg_price){ + ret << third.zhongyuan_avg_price } - if(third.tuboshi_esf_price){ - ret << third.tuboshi_esf_price + if(third.tbs_avg_price){ + ret << third.tbs_avg_price } - if(third.qfang_esf_price){ - ret << third.qfang_esf_price + if(third.qfang_avg_price){ + ret << third.qfang_avg_price } - if(third.fangtan_esf_price){ - ret << third.fangtan_esf_price + if(third.fangtan_avg_price){ + ret << third.fangtan_avg_price } - if(third.fangduoduo_esf_price){ - ret << third.fangduoduo_esf_price + if(third.fangduoduo_avg_price){ + ret << third.fangduoduo_avg_price } - if(third.get("5i5j_esf_price")){ - ret << third.get("5i5j_esf_price") + if(third.wiwj_avg_price){ + ret << third.wiwj_avg_price } if(ret.size()==0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index aaeb0cdef..7062f6c72 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -350,7 +350,7 @@ class LocationTest { mapName = r.name address = r.address - if(r.district.contains(dist.region)){ + if(r.district && r.district.contains(dist.region)){ score = 9 }else { score = 5 @@ -385,7 +385,7 @@ class LocationTest { region = r.district mapName = r.name address = r.address - if(r.district.contains(dist.region)){ + if(r.district && r.district.contains(dist.region)){ score = 7 }else { score = 5 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index 5a8586fb8..fd646bf20 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -35,17 +35,21 @@ class InsidePolygon { public static List createPoint2(String points){ - String[] array = points.split(";"); + List list = new ArrayList(); - for(int i =0 ;i Date: Sat, 7 Aug 2021 14:45:21 +0800 Subject: [PATCH 016/111] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 7 +- .../script/ccbevaluation/TradeStatLog.groovy | 1 + .../script/gj/location/LocationTest.groovy | 134 ++++++++++++++---- 3 files changed, 112 insertions(+), 30 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 7aff7834c..dd345f2d3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -53,10 +53,15 @@ class BigdataCommunityBatch extends BaseBatch{ // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } + def fileName = "/home/goojia/ccb/bigdata_community.txt" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt" + } + def allDatas = [] allDatas << headers allDatas.addAll(rows) - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成" // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index ff50c51f8..b5f10ed40 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -76,6 +76,7 @@ class TradeStatLog { println ret } + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index 7062f6c72..524120aba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point @@ -14,7 +15,13 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql + import java.text.SimpleDateFormat +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern; @@ -22,7 +29,10 @@ import java.util.regex.Pattern; class LocationTest { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { // Date date=new Date(1622802609); // SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @@ -34,6 +44,7 @@ class LocationTest { // task.run2() // task.run3() // task.run4() +// task.run5() } def run(){ @@ -41,40 +52,57 @@ class LocationTest { def regionPointsMap = findRegionRange(city) def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); def index = 0 - def rets = [] - for(def dist : districts){ - def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) - if(!flag){ - index++ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) - def ret = [] - ret << dist.city - ret << dist.region - ret << dist.name + for(def i=0;i + rets << [r.cityid, r.commid] + }); + + def map = [:] + + return map + } + } From 2bf451fb9d952ce05b82765b4b20cad591ff0aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 7 Aug 2021 16:38:01 +0800 Subject: [PATCH 017/111] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/test/AddressStandard.groovy | 2 +- .../main/groovy/com/fangjia/script/util/ExcelUtils.groovy | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AddressStandard.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AddressStandard.groovy index a5cbbabbf..0f4fdd3cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AddressStandard.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AddressStandard.groovy @@ -90,7 +90,7 @@ class AddressStandard { } } } - },0) + },0, "Sheet1") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 1cf782236..cf32d7a34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -60,7 +60,8 @@ public class ExcelUtils{ FileInputStream inp = null; try { inp = new FileInputStream(file); - Workbook workbook = WorkbookFactory.create(inp) + BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 50) + Workbook workbook = WorkbookFactory.create(bufferedInputStream) workbook.setForceFormulaRecalculation(true) Sheet sheet = workbook.getSheetAt(sheetIndex); //行号 @@ -82,6 +83,9 @@ public class ExcelUtils{ } } ret.add(rowData); + if (ret.size() % 10000 == 0) { + println(ret.size()) + } } } catch (IOException e) { e.printStackTrace(); From 457bb5795b6278f1c058e458a160f44bc802a54f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 7 Aug 2021 16:41:39 +0800 Subject: [PATCH 018/111] =?UTF-8?q?=E7=BC=93=E5=86=B2=E5=A2=9E=E5=A4=A7?= =?UTF-8?q?=E4=B8=BA200m?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index cf32d7a34..e6f062685 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -60,7 +60,7 @@ public class ExcelUtils{ FileInputStream inp = null; try { inp = new FileInputStream(file); - BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 50) + BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 200) Workbook workbook = WorkbookFactory.create(bufferedInputStream) workbook.setForceFormulaRecalculation(true) Sheet sheet = workbook.getSheetAt(sheetIndex); From 7dba105573a6b63ba10b8db3ce2174c9a23daddc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 7 Aug 2021 20:42:21 +0800 Subject: [PATCH 019/111] update --- .../BigdataCommunityBatch.groovy | 4 +- .../ccbevaluation/BigdataDealBatch.groovy | 10 +- .../fangjia/script/ccbevaluation/Test.groovy | 180 ++++++++++++++---- .../script/commData/CommonDatautils.groovy | 2 + .../script/gj/location/LocationTest.groovy | 51 +++-- 5 files changed, 185 insertions(+), 62 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index dd345f2d3..b56672f1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -148,11 +148,11 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa100" ret << "" if(dist.address){ - if(dist.address.toString().endsWith("弄")){ + if(dist.address.toString().contains("弄")){ String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄")) if(number){ ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number)) - ret << number + ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size()) }else{ ret << "" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 3a252731e..af0044a94 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -393,14 +393,14 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) - def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] +// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] def startDate = null; - if(citys.contains(city)){ +// if(citys.contains(city)){ startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) - }else{ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) - } +// }else{ +// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// } def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i // if(d['C']){ -// d['P'] = dealMap.get(d['C']) +// d['Q'] = dealMap.get(d['C']) // } //},2) -def city = "中山" -def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) -def index =0 -def updateIndex = 0 -def delIndex = 0 -for(def dist : dists){ - if(dist.region=="石岐区街道"){ - def lat = dist.lat/10000000000.0 - def lng = dist.lng/10000000000.0 - def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) - if(regionStr){ - def region = RegionData.formatRegion(dist.city, regionStr) - def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") - if(obj){ - delIndex++ - DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) - println delIndex+" [del]"+region +" "+dist.name - }else{ - updateIndex++ - DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) - println updateIndex +" [update]"+region+" "+dist.name - } - }else{ - index++ - println index+" [没找到]"+dist.name - } + + +def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" +//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) +//def dealMap = [:] +//for(def d : datas){ +// def map = [:] +// map.put("land_area",d['D']) +// map.put("build_area",d['F']) +// map.put("greeningrate_area",d['H']) +// map.put("parkingcnt_area",d['J']) +// map.put("zts_area",d['L']) +// map.put("buildyear_area",d['N']) +// map.put("plotratio",d['P']) +// +// dealMap.put(d['A'], map) +//} + +def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +ExcelUtils.multiThreadingModify(fileName,{d-> + def city = d['B'] + if(city){ + def cityid = cityCodeDatas.get(city) + def total = findCount(cityid, sqlInstance) + d['I'] = total +// def a = findEstateType(cityid, sqlInstance) +// +// d['AA'] = total +// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +// +// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +// +// def c = findFiledCount(cityid, 'address', sqlInstance) +// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +// +// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +// +// def e = findFiledCount(cityid, 'landarea', sqlInstance) +// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +// +// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +// +// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +// +// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +// +// def i = findFiledCount(cityid, 'latitude', sqlInstance) +// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +// +// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) + + println d } +},0) + +def findEstateType(cityid, sqlInstance){ + def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count } -def getStreetInfo(def city, def glocation){ - def township = null - if(glocation){ - def jsonObject = GaoDeApiUtils.geocode(glocation) - if(jsonObject && jsonObject.regeocode) { - def addressComponent = jsonObject.regeocode.addressComponent - if(addressComponent && addressComponent.city.contains(city)){ - township = addressComponent.township - } - } - } - return township; +def findCount(cityid, sqlInstance){ + def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + +def findFiledCount(cityid, field, sqlInstance){ + def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null"; + def count = 0 + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count } @@ -88,6 +136,54 @@ def getStreetInfo(def city, def glocation){ +//def city = "中山" +//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +//def index =0 +//def updateIndex = 0 +//def delIndex = 0 +//for(def dist : dists){ +// if(dist.region=="石岐区街道"){ +// def lat = dist.lat/10000000000.0 +// def lng = dist.lng/10000000000.0 +// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +// if(regionStr){ +// def region = RegionData.formatRegion(dist.city, regionStr) +// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +// if(obj){ +// delIndex++ +// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) +// println delIndex+" [del]"+region +" "+dist.name +// }else{ +// updateIndex++ +// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +// println updateIndex +" [update]"+region+" "+dist.name +// } +// }else{ +// index++ +// println index+" [没找到]"+dist.name +// } +// } +//} +// +//def getStreetInfo(def city, def glocation){ +// def township = null +// if(glocation){ +// def jsonObject = GaoDeApiUtils.geocode(glocation) +// if(jsonObject && jsonObject.regeocode) { +// def addressComponent = jsonObject.regeocode.addressComponent +// if(addressComponent && addressComponent.city.contains(city)){ +// township = addressComponent.township +// } +// } +// } +// return township; +//} + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 3200b95c6..0338ac33b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -780,6 +780,8 @@ class CommonDatautils { keys.put("address", 1); keys.put("build_structure", 1); keys.put("out_pics", 1); + keys.put("lat", 1); + keys.put("lng", 1); DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index 524120aba..ce1d3ee1a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -29,7 +29,7 @@ import java.util.regex.Pattern; class LocationTest { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -40,9 +40,9 @@ class LocationTest { // System.out.println(str); def task = new LocationTest() - task.run() +// task.run() // task.run2() -// task.run3() + task.run3() // task.run4() // task.run5() } @@ -162,7 +162,9 @@ class LocationTest { def run3(){ - ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d-> + def index = 0 + ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d-> + index++ def distance = null if(d['H']){ distance = d['H'].toDouble() @@ -261,6 +263,7 @@ class LocationTest { } } } + println index },0) } @@ -311,7 +314,7 @@ class LocationTest { } } - def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") if(baiduRest){ bdlocationMap = getMapLocation(dist, baiduRest); if(bdlocationMap){ @@ -464,7 +467,11 @@ class LocationTest { void run() { try{ totalCount.addAndGet(1) - mapLocation(map) + def ret = mapLocation(map) + if(ret){ + rets << ret + } + println totalCount.get() } catch (Exception e) { e.printStackTrace() } finally { @@ -474,14 +481,35 @@ class LocationTest { }); } latch.await() - return rets; + ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误") } def mapLocation(map){ - def dist = CommonDatautils.findDistrictById(map.commid) + def dist = CommonDatautils.findDistrictById(map[1]) if(dist){ - + def loactionMap = matchMap(dist) + def ret = [] + ret << dist.city + ret << dist.region + ret << dist.name + if (loactionMap) { + ret << loactionMap.source + ret << loactionMap.region + ret << loactionMap.mapName + ret << loactionMap.address + def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng) + ret << distance + ret << loactionMap.lng + ret << loactionMap.lat + if (dist.region == loactionMap.region) { + println dist.region + " " + dist.name + " 不在区域范围内,与地图相同" + } else { + println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance + } + } + return ret } + return null } @@ -494,10 +522,7 @@ class LocationTest { sqlInstance.eachRow(sql,{r-> rets << [r.cityid, r.commid] }); - - def map = [:] - - return map + return rets } } From fd263316c7a56c3966ac3570d1450ffa05f4a6e6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Aug 2021 16:01:06 +0800 Subject: [PATCH 020/111] =?UTF-8?q?=E6=B8=A4=E6=B5=B7=E6=98=93=E8=B4=B7?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/dal/mongo/dao/PriceBorderDao.java | 8 ++ .../dal/mongo/dao/PriceBorderDaoImpl.java | 7 ++ .../fangjia/dal/mongo/model/PriceBorder.java | 95 +++++++++++++++++++ .../weixin/WeiXinEvaluationController.java | 87 ++++++++++++++--- .../open/model/weixin/EvaluationModel.java | 12 ++- .../main/webapp/weiXinEvaluation/report.ftl | 24 ++--- .../mongo/fangjia/PriceBorderService.groovy | 8 ++ .../mongo/fangjia/PriceBorderServiceImpl.java | 20 ++++ 8 files changed, 229 insertions(+), 32 deletions(-) create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDao.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDaoImpl.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/PriceBorder.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderServiceImpl.java diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDao.java new file mode 100644 index 000000000..f2748cc26 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDao.java @@ -0,0 +1,8 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.PriceBorder; +import com.fangjia.dal.mongo.model.PriceJiucuo; + +public interface PriceBorderDao extends MongoDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDaoImpl.java new file mode 100644 index 000000000..ff67c4179 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/PriceBorderDaoImpl.java @@ -0,0 +1,7 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.model.PriceBorder; + +public class PriceBorderDaoImpl extends MongoDaoImpl implements PriceBorderDao{ +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/PriceBorder.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/PriceBorder.java new file mode 100644 index 000000000..5fefd029e --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/PriceBorder.java @@ -0,0 +1,95 @@ +package com.fangjia.dal.mongo.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; + +@Entity +@Table(name = "fangjia_crawl.price_border") +public class PriceBorder implements MongoEntity { + + private String id; + private String city; + private String region; + private String name; + private Double latitude; + private Double longitude; + private String border; + private Integer level; + + @Id + @GeneratedValue + @Column(name = "_id", unique = true, nullable = false) + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + @Column(name = "latitude") + public Double getLatitude() { + return latitude; + } + + public void setLatitude(Double latitude) { + this.latitude = latitude; + } + + @Column(name = "longitude") + public Double getLongitude() { + return longitude; + } + + public void setLongitude(Double longitude) { + this.longitude = longitude; + } + + @Column(name = "border") + public String getBorder() { + return border; + } + + public void setBorder(String border) { + this.border = border; + } + + @Column(name = "level") + public Integer getLevel() { + return level; + } + + public void setLevel(Integer level) { + this.level = level; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 9d6ee6cbe..33009b27b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -2,14 +2,13 @@ package com.fangjia.open.controllers.weixin; import java.text.DecimalFormat; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; +import com.fangjia.service.mongo.fangjia.PriceBorderService; import net.sf.json.JSONArray; import net.sf.json.JSONObject; @@ -25,6 +24,7 @@ import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.AverageStatistics; import com.fangjia.dal.mongo.model.Evaluation; import com.fangjia.dal.mongo.model.LatlngPolygon; +import com.fangjia.dal.mongo.model.PriceBorder; import com.fangjia.dal.mongo.model.bkoff.DistrictHouseStat; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.EvaluationAudit; @@ -105,6 +105,7 @@ public class WeiXinEvaluationController { private EvaluationRateService evaluationRateService; private EvaluationService evaluationService; private GoldLogsService goldLogsService; + private PriceBorderService priceBorderService; private final static String DD_ID = "5938c43984aec20a36f474c5"; @@ -369,15 +370,20 @@ public class WeiXinEvaluationController { FjtUsers fjtUser = fjtUsersService.findById(evaluation.getUserId()); - model.setRole(fjtUser.getRole()); - + model.setRole(fjtUser.getRole()); + District district = districtService.findDistrict(new DistrictKey(evaluation.getCity(), evaluation.getRegion(), evaluation.getDistrictName()), "district"); model.setId(district.getId()); //参考价 EvaluationAudit evaluationAudit = evaluationAuditService.findByEvalutionId(evaluationId); model.setEvaluationAudit(evaluationAudit); - - model.setRole(fjtUser.getRole()); + + if(fjtUser!=null && fjtUser.getRole()==17 && district.getCity().equals("上海")){ + Map retMap = getLoanInfo(district, evaluation.getHouseArea()); + if(retMap!=null && retMap.size()>0){ + model.setLoanInfoMap(retMap); + } + } return model; } @@ -790,7 +796,62 @@ public class WeiXinEvaluationController { } return loopLine; } - + + private Map getLoanInfo(District district, Double area){ + List ranges = priceBorderService.getALLBlockRange(); + PriceBorder pbObj = null; + for(PriceBorder pb : ranges){ + String[] pointArrs = pb.getBorder().split(";"); + List points = new ArrayList(); + for(String pointStr : pointArrs){ + String[] ll = pointStr.split(","); + Point point = new Point(Double.valueOf(ll[0]), Double.valueOf(ll[1])); + points.add(point); + } + Point point = new Point(district.getLng2()/10000000000.0, district.getLat2()/10000000000.0); + boolean flag = InsidePolygon.aboutZone(point, points); + if(flag){ + pbObj = pb; + break; + } + } + Integer yearAge = getBuildYear(district.getFinishBuildingDate()); + Map retsMap = new HashMap(); + if(pbObj!=null && pbObj.getLevel()!=null && pbObj.getLevel()==1 && yearAge!=null && yearAge<=25 && area>=60){ + retsMap.put("type", "A"); + retsMap.put("loan_ratio", 7); + }else if(pbObj!=null && pbObj.getLevel()!=null && pbObj.getLevel()==2 && yearAge!=null && yearAge<=25 && area>=70){ + retsMap.put("type", "B"); + retsMap.put("loan_ratio", 6.5); + }else if(yearAge!=null && yearAge<=25 && yearAge>20 && pbObj!=null && pbObj.getLevel()!=null && (pbObj.getLevel()==1 || pbObj.getLevel()==2)){ + retsMap.put("type", "C"); + retsMap.put("loan_ratio", 6); + }else if(yearAge!=null && yearAge<=20 ){ + retsMap.put("type", "C"); + retsMap.put("loan_ratio", 6); + }else if(yearAge!=null && yearAge>25){ + retsMap.put("type", "D"); + retsMap.put("loan_ratio", 5); + } + return retsMap; + } + + public Integer getBuildYear(String finishBuildingDate){ + if(StringUtils.isNotBlank(finishBuildingDate)){ + Pattern pat3 = Pattern.compile("\\d{4}"); + Matcher matcher3 = pat3.matcher(finishBuildingDate); + if (matcher3.find()) { + String year = matcher3.group(); + if(StringUtils.isNotBlank(year)){ + int curYear = Calendar.getInstance().get(Calendar.YEAR); + return curYear - Integer.valueOf(year); + } + } + } + return null; + } + + public void setFjtEvaluationService(FjtEvaluationService fjtEvaluationService) { this.fjtEvaluationService = fjtEvaluationService; @@ -868,6 +929,8 @@ public class WeiXinEvaluationController { public void setEmailService(EmailService emailService) { this.emailService = emailService; } - - + + public void setPriceBorderService(PriceBorderService priceBorderService) { + this.priceBorderService = priceBorderService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java index dcdb3e2c2..1282b227f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java @@ -2,6 +2,7 @@ package com.fangjia.open.model.weixin; import com.fangjia.dal.mongo.model.Evaluation; import java.util.List; +import java.util.Map; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.EvaluationAudit; @@ -38,6 +39,7 @@ public class EvaluationModel extends BaseModel{ private String userType; private Integer role; private String cardId; + private Map loanInfoMap; public void setId(String id) { @@ -186,6 +188,12 @@ public class EvaluationModel extends BaseModel{ public void setCardId(String cardId) { this.cardId = cardId; } - - + + public Map getLoanInfoMap() { + return loanInfoMap; + } + + public void setLoanInfoMap(Map loanInfoMap) { + this.loanInfoMap = loanInfoMap; + } } diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/report.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/report.ftl index d40ad0c4e..0b2fb5b13 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/report.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/report.ftl @@ -31,23 +31,11 @@

璇勪及鎬讳环锛�<#if fjtEvaluation??>${(fjtEvaluation.totalPrice/10000)?string('0.##')} 涓囧厓

璇勪及鍗曚环锛�${fjtEvaluation.averagePrice!} 鍏�/骞崇背

- <#if role?? && role==17> - <#if fjtEvaluation.districtName=="閿︾唬鑺辨湪鍏瘬"> -

鎵�灞炲垎绫伙細A绫�

-

鍙捶鎴愭暟锛�7鎴�

-

鍙捶閲戦锛�${(fjtEvaluation.totalPrice*0.7/10000)?string('0.##')}涓囧厓

+ <#if role?? && role==17 && loanInfoMap??> +

鎵�灞炲垎绫伙細${loanInfoMap.type!}绫�

+

鍙捶鎴愭暟锛�${loanInfoMap.loan_ratio!}鎴�

+

鍙捶閲戦锛�${(fjtEvaluation.totalPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

璐锋鍒╃巼锛�4.5%

- <#elseif fjtEvaluation.districtName=="鍗楀北闆ㄦ灉"> -

鎵�灞炲垎绫伙細C绫�

-

鍙捶鎴愭暟锛�6鎴�

-

鍙捶閲戦锛�${(fjtEvaluation.totalPrice*0.6/10000)?string('0.##')}涓囧厓

-

璐锋鍒╃巼锛�4.5%

- <#elseif fjtEvaluation.districtName=="鏂版尘涓夋潙"> -

鎵�灞炲垎绫伙細D绫�

-

鍙捶鎴愭暟锛�5鎴�

-

鍙捶閲戦锛�${(fjtEvaluation.totalPrice*0.5/10000)?string('0.##')}涓囧厓

-

璐锋鍒╃巼锛�4.5%

-
@@ -56,7 +44,7 @@
- <#if role?? && (role==12 || role==16)> + <#if role?? && (role==12 || role==16 || role==17)> <#if evaluationAudit?? && evaluationAudit.auditPrice??>
@@ -93,7 +81,7 @@ - <#if role?? && (role==12 || role==16)> + <#if role?? && (role==12 || role==16 || role==17)>
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy new file mode 100644 index 000000000..bb303df5d --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy @@ -0,0 +1,8 @@ +package com.fangjia.service.mongo.fangjia + +import com.fangjia.dal.mongo.model.PriceBorder + +interface PriceBorderService { + + public List getALLBlockRange(); +} \ No newline at end of file diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderServiceImpl.java new file mode 100644 index 000000000..72ffa0a09 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderServiceImpl.java @@ -0,0 +1,20 @@ +package com.fangjia.service.mongo.fangjia; + +import com.fangjia.dal.mongo.dao.PriceBorderDao; +import com.fangjia.dal.mongo.model.PriceBorder; + +import java.util.List; + +public class PriceBorderServiceImpl implements PriceBorderService{ + + private PriceBorderDao priceBorderDao; + + public void setPriceBorderDao(PriceBorderDao priceBorderDao) { + this.priceBorderDao = priceBorderDao; + } + + @Override + public List getALLBlockRange() { + return priceBorderDao.findAll(); + } +} From c8c53e85ae43036598f1b8df6e5f22ec222457c4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Aug 2021 16:07:14 +0800 Subject: [PATCH 021/111] update --- .../com/fangjia/service/mongo/fangjia/PriceBorderService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy index bb303df5d..fd30671da 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy @@ -4,5 +4,5 @@ import com.fangjia.dal.mongo.model.PriceBorder interface PriceBorderService { - public List getALLBlockRange(); + List getALLBlockRange(); } \ No newline at end of file From 90a3b70da6ab7ecc441c2ff097306c0873f2933e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Aug 2021 16:19:55 +0800 Subject: [PATCH 022/111] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/mongo/fangjia/PriceBorderService.groovy | 8 -------- .../service/mongo/fangjia/PriceBorderService.java | 9 +++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.java diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy deleted file mode 100644 index fd30671da..000000000 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.groovy +++ /dev/null @@ -1,8 +0,0 @@ -package com.fangjia.service.mongo.fangjia - -import com.fangjia.dal.mongo.model.PriceBorder - -interface PriceBorderService { - - List getALLBlockRange(); -} \ No newline at end of file diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.java new file mode 100644 index 000000000..a37cdaf62 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/PriceBorderService.java @@ -0,0 +1,9 @@ +package com.fangjia.service.mongo.fangjia; + +import com.fangjia.dal.mongo.model.PriceBorder; + +import java.util.List; + +public interface PriceBorderService { + List getALLBlockRange(); +} From e4e08ac004721d5764dfa42f01eea47c1b4ac5c5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 12 Aug 2021 19:00:18 +0800 Subject: [PATCH 023/111] update --- .../ccbevaluation/BigdataDealBatch.groovy | 57 +++++++++++++++---- .../ccbevaluation/CcbCommonUtils.groovy | 31 +++++----- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +- 3 files changed, 65 insertions(+), 29 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index af0044a94..5d7d83944 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -28,12 +28,13 @@ import java.util.regex.Pattern class BigdataDealBatch extends BaseBatch{ def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static main(args) { runBatch(new BigdataDealBatch(), args); @@ -41,6 +42,7 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { + def weekend = DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; def rows = [] def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] @@ -48,13 +50,14 @@ class BigdataDealBatch extends BaseBatch{ for(def city : citys){ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def trades = getCityData(city) + def statMap = getDistrictStatByCity(city, weekend) + def trades = getCityData(city, statMap) def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") rows.addAll(rets) - println city +" 成交共:"+rets.size() + if(city=="上海"){ - def youDataTrade = getYouDaData() + def youDataTrade = getYouDaData(statMap) def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") rows.addAll(youdaDatas) println city +" "+youdaDatas.size() @@ -386,7 +389,7 @@ class BigdataDealBatch extends BaseBatch{ return s; } - def getCityData(def city){ + def getCityData(def city, def statMap){ def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","build_type","build_structure", "house_property","property_life"]) def tradesList = Collections.synchronizedList(new ArrayList()); @@ -405,11 +408,18 @@ class BigdataDealBatch extends BaseBatch{ def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i=startPrice && obj.price<=endPrice){ + rets << obj; + } + + } + } } return rets; } + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = [:] + for(def d : dbcursor){ + map.put(d.region+"@"+d.district_name, d.avg_price) + } + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index cff67d153..f7358c436 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -12,7 +12,8 @@ import com.mongodb.DBObject class CcbCommonUtils { - public static def citys = ["上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + + public static def citys = ["扬州","上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -59,7 +60,7 @@ class CcbCommonUtils { return rets } - public static def findTradesDetail(dist, startDate, endDate){ + public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){ def q = new BasicDBObject() q.put("city", dist.city) q.put("region", dist.region) @@ -83,18 +84,20 @@ class CcbCommonUtils { def rets = new HashSet() def lists = [] for(def d : dbcursor){ - def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") - if(!rets.contains(key)){ - rets << key - d.dist_id = dist._id.toString() - d.address = dist.address - d.master_name = dist.name - d.build_type = dist.build_type - d.build_structure = dist.build_structure - d.house_roperty = dist.house_roperty - d.property_life = dist.property_life - d.estate_type2 = dist.estate_type2 - lists << d + if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } } } return lists diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index a81e03ba1..55fb15bd5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -18,7 +18,7 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -73,7 +73,7 @@ class ThirdPartyPriceBatch extends BaseBatch { ret << d.city ret << data.district_code ret << data.district - ret << third.s_date + ret << sDate if(third.zhugezhaofang_esf_price){ ret << third.zhugezhaofang_esf_price @@ -169,7 +169,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city){ def q = new BasicDBObject() q.put("city", city) - q.put("s_date", new BasicDBObject('$gte', curDate)) + q.put("s_date", new BasicDBObject('$gte', sDate)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ From f3f790d65b232d0d30a5489ab1c4e9ed61639e7a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 13 Aug 2021 09:54:01 +0800 Subject: [PATCH 024/111] update --- .../ccbevaluation/BigdataDealBatch.groovy | 5 +- .../ccbevaluation/CcbCommonUtils.groovy | 3 +- .../com/fangjia/script/gj/FiledStrand.groovy | 55 ++++ .../gj/data/AddDistrictFieldValue.groovy | 268 ++++++++++++++++++ 4 files changed, 326 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 5d7d83944..2a413d1b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -28,7 +28,7 @@ import java.util.regex.Pattern class BigdataDealBatch extends BaseBatch{ def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -265,9 +265,8 @@ class BigdataDealBatch extends BaseBatch{ def ret = [] def data = distCodeMap.get(trade.region) if(!data){ - return ret - }else{ println trade.city +" "+trade.region+" 没有找到对应的区域" + return ret } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index f7358c436..5a74c2733 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -12,8 +12,7 @@ import com.mongodb.DBObject class CcbCommonUtils { - - public static def citys = ["扬州","上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + public static def citys = [ "扬州","上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 3fd9abe32..cdbeaefd7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -259,6 +259,31 @@ class FiledStrand { return updateFile; } + static def processArea(all_area){ + if(all_area){ + def v = getValue(all_area) + if(v>=1000 && v<=10000000){ + + return v.toString() + } + } + return null + } + + static def processCarSite(car_site_count){ + def v = getValue(car_site_count.toString()); + if(v){ + if(v>=1 && v<=100000){ + try { + return v.toString().toInteger(); + }catch(Exception e){ + println e.getMessage() + } + } + } + return null + } + static def addBuildAreaToData(def dist, def updateFile){ def build_area = null; if(dist.build_area){ @@ -298,6 +323,26 @@ class FiledStrand { } return updateFile; } + + public static def processCubagePercent(cubage_percent, estate_type2){ + if(cubage_percent){ + def v = getValue(cubage_percent) + if(v){ + if(estate_type2 == "别墅"){ + if(v>=0.3 && v<=1){ + return v.toString() + } + }else{ + if(v>1 && v<=10){ + return v.toString() + } + } + } + } + return null + } + + static def addVirescencePercentToData(dist, updateFile){ def virescence_percent = null if(dist.virescence_percent){ @@ -312,6 +357,16 @@ class FiledStrand { return updateFile; } + static def processVirescencePercent(virescence_percent){ + if(virescence_percent){ + def v = getValue(virescence_percent) + if(v && v>15 && v<=70){ + return v.toString() + } + } + return null; + } + static def addBuildTypeToData(dist, updateFile){ if(dist.estateType2 && dist.buildType){ def house = ["超高层","小高层","多层","高层","花园洋房","老洋房","四合院"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy new file mode 100644 index 000000000..d91c34642 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy @@ -0,0 +1,268 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import com.rabbitmq.client.AMQP +import com.rabbitmq.client.impl.AMQImpl +import org.bson.types.ObjectId + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AddDistrictFieldValue { + + def detailColl = CrawalDBUtil.getDistrictDetailColl(); + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + def task = new AddDistrictFieldValue(); + task.run(); + } + + def run(){ + def citys = CcbCommonUtils.citys; +// for(def city : citys){ +// def rets = findAllDataByCity(city) +// match(rets) +// } + + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) + def rMap = getMatchDistrict(city) + createResultValue(dists, rMap) + } + + } + + def match(def rets){ + def totalCount = new AtomicLong() + def total = rets.size() + final CountDownLatch latch = new CountDownLatch(total) + for(def i=0;i0){ + updateIndex++ + def q = new BasicDBObject() + q.put("_id", d._id) + println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + + } + } + } + } + + def getValuesByField(sourceMap, field, estateType){ + def map = [:] + sourceMap.each{k,v-> + def oldValue = v.get(field); + if(oldValue){ + def newValue = null; + if(["address"].contains(field)){ + newValue = DataUtil.formateAddress(v.get("city"), oldValue) + } + if(["all_area","build_area"].contains(field)){ + newValue = FiledStrand.processArea(oldValue) + } + if(["build_date"].contains(field)){ + newValue = FiledStrand.processFinishBuildingDate(oldValue) + } + if(["house_count"].contains(field)){ + newValue = FiledStrand.processHouseholdCount(oldValue, estateType) + } + if(["greening_rate"].contains(field)){ + newValue = FiledStrand.processVirescencePercent(oldValue) + } + if(["volumetric_rate"].contains(field)){ + newValue = FiledStrand.processCubagePercent(oldValue, estateType) + } + if(["car_site_desc","car_site"].contains(field)){ + newValue = FiledStrand.processCarSite(oldValue) + } + if(newValue){ + def count = map.get(newValue) + if(!count){ + map.put(newValue, 1) + }else{ + map.put(newValue, count+1) + } + + v.put(field, newValue) + } + } + } + + def lastValue = null + if(map && map.size()>0){ + map = map.sort {a, b -> b.value <=> a.value} + def values = map.values() + if(values.size()==1){ + lastValue = map.keySet()[0] + }else{ + def count1 = map.values()[0] + def count2 = map.values()[1] + if(count1 > count2){ + lastValue = map.keySet()[0] + }else{ + def rets = [map.keySet()[0],map.keySet()[1]] + def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"] + for(def root : roots){ + def data = sourceMap.get(root) + if(data){ + def value = data.get(field) + if(rets.contains(value)){ + lastValue = value + break + } + } + } + } + } + } + + return lastValue; + } + + +} From 1ea267b81fcfec82199630cabd9e18a09ac52005 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 13 Aug 2021 16:29:01 +0800 Subject: [PATCH 025/111] update --- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/EvaluateRateBatch.groovy | 37 ++- .../fangjia/script/ccbevaluation/Test.groovy | 282 ++++++++++++++---- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 4 +- 5 files changed, 250 insertions(+), 79 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index b56672f1d..560facf40 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -44,7 +44,7 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" 小区量:"+dists.size() def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index 5a0c7a328..d64c08dd6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -36,21 +36,36 @@ class EvaluateRateBatch extends BaseBatch{ def createDatas(def datas){ def allRows = [] datas.each { m -> - def data = [] - data << m.id.toString() - data << m.rate_type - data << m.rate - data << m.rank - data << "全国" - data << m.rank_remark - data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - data << "FJW" - allRows << data + if(m.rank_remark!="开发商装修"){ + def data = [] + data << m.id.toString() + data << m.rate_type + data << m.rate + data << m.rank + data << "全国" + data << converFitment(m.rank_remark) + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + allRows << data + } } return allRows } + private def converFitment(rank_remark){ + if(rank_remark){ + if(rank_remark.contains("简单装修")){ + return "简装" + }else if(rank_remark.contains("精装修")){ + return "精装" + }else if(rank_remark.contains("豪华装修")){ + return "豪装" + } + } + return rank_remark + } + private def findEvaluationRate(){ def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'"; def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index f1e86c6e9..99b243d36 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -3,15 +3,35 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.util.regex.*; def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + +//String str="\${ }"; +//String regex="\\\$\\{.*\\}"; +//Matcher m=Pattern.compile(regex).matcher(str); +//System.out.println(m.matches()); + + + + + + //def citys = CcbCommonUtils.citys; //def rows = [] // @@ -43,7 +63,7 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", -def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" +//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def dealMap = [:] //for(def d : datas){ @@ -59,48 +79,102 @@ def fileName = "d:\\ // dealMap.put(d['A'], map) //} -def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -ExcelUtils.multiThreadingModify(fileName,{d-> - def city = d['B'] - if(city){ - def cityid = cityCodeDatas.get(city) - def total = findCount(cityid, sqlInstance) - d['I'] = total -// def a = findEstateType(cityid, sqlInstance) +//def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//ExcelUtils.multiThreadingModify(fileName,{d-> +// def city = d['B'] +// if(city){ +// def cityid = cityCodeDatas.get(city) +// def total = findCount(cityid, sqlInstance) +//// d['I'] = total +//// def a = findEstateType(cityid, sqlInstance) +//// +//// d['AA'] = total +//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +//// +//// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +//// +//// def c = findFiledCount(cityid, 'address', sqlInstance) +//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +//// +//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +//// +//// def e = findFiledCount(cityid, 'landarea', sqlInstance) +//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +//// +//// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +//// +//// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +//// +//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +//// +//// def i = findFiledCount(cityid, 'latitude', sqlInstance) +//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +//// +//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) // -// d['AA'] = total -// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) -// -// def b = findFiledCount(cityid, 'buildyear', sqlInstance) -// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) -// -// def c = findFiledCount(cityid, 'address', sqlInstance) -// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) -// -// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) -// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) -// -// def e = findFiledCount(cityid, 'landarea', sqlInstance) -// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) -// -// def f = findFiledCount(cityid, 'buildarea', sqlInstance) -// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) -// -// def g = findFiledCount(cityid, 'plotratio', sqlInstance) -// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) -// -// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) -// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) -// -// def i = findFiledCount(cityid, 'latitude', sqlInstance) -// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) -// -// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) -// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +// println d +// } +//},0) +//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx" +//def dealMap = findDealCount(sqlInstance) +//ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['C']){ +// d['Q'] = dealMap.get(d['C']) +// } +//},2) + + +//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +//def updateIndex = 0 +//def deleteIndex = 0 +//ExcelUtils.modify(fileName,{d-> +// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ +// def dist = CommonDatautils.findDistrictById(d['D']) +// def city = dist.city +// def region = RegionData.formatRegion(city, d['Q']) +// if(region && dist.region!=region){ +// println city +" new:"+region +" old:"+d['C'] +// def udata = new BasicDBObject() +// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +// if(obj){ +// deleteIndex++ +// udata.put("status", 1) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +// }else{ +// updateIndex++ +// udata.put("region", region) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +// } +// }else{ +// println d['D']+" 不存在" +// } +// } +//},0) + + +def findDealCount(sqlInstance){ + def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate + map.put(r.cityname, r.count) + }); + return map +} + + - println d - } -},0) def findEstateType(cityid, sqlInstance){ def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'"; @@ -132,22 +206,95 @@ def findFiledCount(cityid, field, sqlInstance){ } +def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +def updateIndex = 0 +def delIndex = 0 +def cityMap = [:] +ExcelUtils.multiThreadingModify(fileName,{d-> + if(!d['R']){ + + def obj = CommonDatautils.findDistrictById(d['D']) + if(obj){ + def ranges = cityMap.get(obj.city) + if(!ranges){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) + ranges = [] + for(def r : dbcursor){ + ranges << r + } + cityMap.put(obj.city, ranges) + } + + if(ranges){ + def lng = obj.lng/10000000000.0 + def lat = obj.lat/10000000000.0 + + def region = null + for(def r : ranges){ + try{ + def flag = isInArea(lng, lat, r.polyline) + if(flag){ + def regionStr = r.region + region = RegionData.formatRegion(obj.city, regionStr) + break + } + }catch(Exception e){ + println e.getMessage() + } + + } + if(region && region!=obj.region){ + updateIndex++ + DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) + println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name + }else if(region && region==obj.region){ + println "[same] "+obj.city +" "+obj.name + }else{ + DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) + delIndex++ + println delIndex +" [delete] "+obj.city +" "+obj.name + } + }else{ + println obj.city +" 没有此城市边框信息" + } + + }else{ + println d['D']+" 没找到该小区" + } + + } +},0); - -//def city = "中山" +//def city = "杭州" //def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) +//def ranges = [] +//for(def d : dbcursor){ +// ranges << d +//} //def index =0 //def updateIndex = 0 //def delIndex = 0 //for(def dist : dists){ -// if(dist.region=="石岐区街道"){ +//// if(dist.region=="江干"){ // def lat = dist.lat/10000000000.0 // def lng = dist.lng/10000000000.0 -// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) -// if(regionStr){ -// def region = RegionData.formatRegion(dist.city, regionStr) +// +// def regionStr =null +// def region = null +// for(def r : ranges){ +// def flag = isInArea(lng, lat, r.polyline) +// if(flag){ +// regionStr = r.region +// region = RegionData.formatRegion(dist.city, regionStr) +// break +// } +// } +// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +// if(regionStr && region!=dist.region){ +// // def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") // if(obj){ // delIndex++ @@ -156,30 +303,39 @@ def findFiledCount(cityid, field, sqlInstance){ // }else{ // updateIndex++ // DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]"+region+" "+dist.name +// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name // } // }else{ // index++ // println index+" [没找到]"+dist.name // } -// } -//} -// -//def getStreetInfo(def city, def glocation){ -// def township = null -// if(glocation){ -// def jsonObject = GaoDeApiUtils.geocode(glocation) -// if(jsonObject && jsonObject.regeocode) { -// def addressComponent = jsonObject.regeocode.addressComponent -// if(addressComponent && addressComponent.city.contains(city)){ -// township = addressComponent.township -// } -// } -// } -// return township; +//// } //} +def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + //Point point = new Point(lng, lat); + //def flag = InsidePolygon.aboutZone(point, pointList) + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; +} + +def getStreetInfo(def city, def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent && addressComponent.city.contains(city)){ + township = addressComponent.township + } + } + } + return township; +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 55fb15bd5..179daf27e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -186,8 +186,8 @@ class ThirdPartyPriceBatch extends BaseBatch { if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){ dataMap.put("anjuke_esf_price", db.anjuke_esf_price) } - if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf")){ - dataMap.put("anjuke_esf_price", db.get("58tc_esf")) + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) } if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){ dataMap.put("soufang_esf_price", db.soufang_esf_price) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 0338ac33b..ca7dada90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -218,7 +218,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","东寓新村") +// q.put("name","金辉伯爵山") DBObject keys = new BasicDBObject(); if(fields){ @@ -1316,7 +1316,7 @@ class CommonDatautils { } public static def getAllDictionary(){ - def dbCursor = DICT_COLL.find(new BasicDBObject()) + def dbCursor = DICT_COLL.find(new BasicDBObject()).batchSize(5000) def rets = [] def filters = [ "estate","hfword","main","metro","station"] for(def obj : dbCursor){ From 9926e84363b3eb0c217333bb4674dfc719941e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 16 Aug 2021 14:43:46 +0800 Subject: [PATCH 026/111] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=B0=E5=80=BC?= =?UTF-8?q?=E6=8A=A5=E5=91=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/weixin/CcbRentController.java | 3 ++- .../src/main/webapp/ccbRent/detail.ftl | 22 ++++++++++++++----- .../src/main/webapp/ccbRent/downloadPage.ftl | 4 ++-- .../src/main/webapp/ccbRent/index.ftl | 2 +- .../src/main/webapp/ccbRent/report.ftl | 2 +- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java index 3a17ffc8b..38e9056f1 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java @@ -50,6 +50,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.*; import java.lang.reflect.InvocationTargetException; +import java.net.URLEncoder; import java.text.ParseException; import java.util.*; @@ -487,7 +488,7 @@ public class CcbRentController { @Param(name = "name") String name, @Param(name = "address") String address, @Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room, @Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException { - response.setHeader("Content-disposition", "attachment; filename=report.pdf"); + response.setHeader("Content-disposition", "attachment; filename=" + URLEncoder.encode("租金估值报告", "UTF-8") + ".pdf"); response.setContentType("application/pdf;charset=UTF-8"); response.setCharacterEncoding("UTF-8"); OutputStream os = null; diff --git a/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl b/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl index e606e1324..b896cb481 100644 --- a/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl +++ b/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl @@ -12,7 +12,7 @@ - + @@ -23,13 +26,13 @@