From 7dff8154c5a449b5f73f80fed61dfb6e93b60c96 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 17 Jul 2021 20:54:44 +0800 Subject: [PATCH 1/3] add ccb batch file --- .../BigdataCommunityBatch.groovy | 154 ++++++ .../ccbevaluation/BigdataDistrictBatch.groovy | 91 +++ .../ccbevaluation/BigdataForsaleBatch.groovy | 520 +++++++++++++++++ .../ccbevaluation/BigdataForsaleBatch2.groovy | 521 ++++++++++++++++++ .../ccbevaluation/CcbCommonUtils.groovy | 61 ++ .../script/ccbevaluation/CityBatch.groovy | 100 ++++ .../ccbevaluation/EvaluateRateBatch.groovy | 64 +++ .../ccbevaluation/GuiDingPriceBatch.groovy | 99 ++++ .../ccbevaluation/StagingDistrictBatch.groovy | 85 +++ .../ccbevaluation/ThirdPartyPriceBatch.groovy | 177 ++++++ .../gj/data/BestDistrictPriceExport.groovy | 174 ++++++ .../gj/district/SourceDistDataMatch.groovy | 265 +++++++++ .../script/gj/region/MapAreaUpdate.groovy | 65 +++ 13 files changed, 2376 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy new file mode 100644 index 000000000..3913caee7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -0,0 +1,154 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 建行模型估值项目 + * 小区数据同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class BigdataCommunityBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new BigdataCommunityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + + def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + "总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + "乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] + + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) + def rets = createCityDatas(cityCodeDatas, distCodeMap, dists) + rows.addAll(rets) + println city +" "+rets.size() + + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); + } + + def allDatas = [] + allDatas << headers + allDatas.addAll(rows) + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false); + println "txt 生成" + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") + } + + def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){ + def rets = [] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + def data = [] + data << m.dist_code + data << m.dist_name + data << m.dist_spell + data << m.city_id + data << m.dist_code + data << index + data << "" + data << "" + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + data << "" + data << "" + data << "" + data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + index++ + allRows << data + } + return allRows; + } + + def findDistrictCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + q.put("district", new BasicDBObject('$ne',"市辖区")) + def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + def lists = [] + for(def db : dbcursor){ + if(db.district_code){ + def m = [:] + m.id = db._id.toString() + m.dist_name = db.fangjia_district + m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") + m.city_id = db.city_code + m.dist_code = db.district_code + lists << m + } + } + return lists; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy new file mode 100644 index 000000000..7e97743c6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -0,0 +1,520 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataForsaleBatch extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + + static main(args) { + runBatch(new BigdataForsaleBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731); + } + } + + def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){ + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println city +" 小区量:"+districts.size() + + def retMap = new ConcurrentHashMap() + for(def i=0;i0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = city + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(rootMap.get(seawater.root)){ + rowMap.root = rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + } + } + return rets; + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + /* + * 生成一个用不重复的ID + */ + def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) + } + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy new file mode 100644 index 000000000..c896476d1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy @@ -0,0 +1,521 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataForsaleBatch2 extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + + def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + + static main(args) { + runBatch(new BigdataForsaleBatch2(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def distMap = getDistrictMap(city) + exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731); + } + } + + def getDistrictMap(def city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + if(d.alias){ + d.alias.each{name-> + if(!distMap.containsKey(d.region+"##"+name)){ + distMap.put(d.region+"##"+name, d) + } + } + } + } + return distMap; + } + + def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){ + println city +" 小区key:"+distMap.size() + def cityCode = cityCodeDatas.city_code + def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap) + def allDatas = createSellDatas(city, retMap); + def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true); + println "txt文件生成 " + } + + + + def createSellDatas(def city, def map){ + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(map.size()) + println city +"总房源量:"+map.size() + + def datas = [] + map.each{k,m-> + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + datas << converData(m) + println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return datas ; + } + + def converData(def m){ + def data = [] + data << m.city_code + data << m.houseId + data << "" + data << m.city + data << m.region_code + data << m.region + data << m.district_id + data << m.district_name + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << m.address==null?"":m.address==null + data << m.build_year==null?"":m.build_year + data << m.proptype==null?"":m.proptype + data << m.propyears==null?"":m.propyears + data << m.avg_price==null?"":m.avg_price + data << m.total_price==null?"":m.total_price + data << "" + data << m.floor==null?"":m.floor + data << m.height==null?"":m.height + data << m.estate_type==null?"":m.estate_type + data << m.house_structure_code==null?"":m.house_structure_code + data << m.build_structure_code==null?"":m.build_structure_code + data << m.area==null?"":m.area + data << "" + data << m.house_type==null?"":m.house_type + data << m.room==null?"":m.room + data << m.hall==null?"":m.hall + data << "" + data << m.toilet==null?"":m.toilet + data << m.direction_code==null?"":m.direction_code + data << m.fitment_code==null?"":m.fitment_code + data << "" + data << "" + data << m.p_date==null?"":m.p_date + data << "" + data << "" + data << m.root + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << m.c_date==null?"":m.c_date + data << m.m_date==null?"":m.m_date + data << m.source + data << "" + return data + } + + + def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def retMap = new ConcurrentHashMap() + println city +" 没过滤前:"+ houses.size() + if(houses){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(houses.size()) + + for (int k=0;k60){ + return null + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + return null + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + return null + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + return null + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + return null + } + + if(seawater.avg_price && seawater.avg_price<10){ + return null + } + + } + rowMap.houseId = seawater.id + rowMap.city = city + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(rootMap.get(seawater.root)){ + rowMap.root = rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + + def regionCode = distCodeMap.get(rMap.region) + def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) + def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) + def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) + def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type); + def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); + rowMap.city_code = cityCode + rowMap.region = rMap.region + rowMap.region_code = regionCode + rowMap.district_id = rMap.get("_id").toString() + rowMap.district_name = rMap.name + rowMap.build_year = buildYear; + rowMap.proptype = housePro + rowMap.propyears = propertyYear; + rowMap.estate_type = estateType + rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall); + rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction) + rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment); + rowMap.address = rMap.address + rowMap.build_structure_code = struCode + rowMap.house_structure_code = null//getHouseStructureCode(); + rowMap.source = "FJW" + return rowMap + + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + /* + * 生成一个用不重复的ID + */ + def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) + } + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy new file mode 100644 index 000000000..37c99bc96 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CcbCommonUtils { + + public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] +// public static def citys = ["北京"] + + public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] + + public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") + + public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") + + public static def findCityCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count >1){ + def data = null + for(def db : dbcursor){ + if(db.city_code){ + data = db + break + } + } + return data; + } + } + + public static def findAllDistrictCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + q.put("district", new BasicDBObject('$ne',"市辖区")) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def map = [:] + for(def db : dbcursor){ + map.put(db.fangjia_district, db.district_code) + } + return map; + } + + public static def findPersonPrice(city, date){ + def q = new BasicDBObject() + q.put("city", city) + q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date))); + def dbcursor = GUIDING_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy new file mode 100644 index 000000000..bafa9e4d8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +/** + * 建行模型估值项目 + * 城市数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class CityBatch extends BaseBatch{ + + static main(args) { + runBatch(new CityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + def index = 1 + for(def city : citys){ + def data = CcbCommonUtils.findCityCodeValue(city) + def map = [:] + map.name = city + if(data){ + map.city_code = data.city_code + map.prov_code = data.province_code + } + def cityObj = findCityInfo(city) + if(cityObj){ + map.city_spell = cityObj.spell + map.city_level = cityObj.city_grade + map.seq = index + map.ifcapital = isCapital(cityObj.province, city) + map.ifmunicipal = ifMunicipal(city) + } + index++ + rows << map + } + + def allDatas = createDatas(rows) + def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"] + + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8"); + } + + def createDatas(rows){ + def allRows = [] + rows.each { m -> + def data = [] + data << m.city_code + data << m.name + data << m.city_spell + data << m.prov_code + data << m.city_code + data << m.city_level + data << m.seq + data << CcbCodeUtils.parseFlag(m.ifcapital) + data << CcbCodeUtils.parseFlag(m.ifmunicipal) + data << "" + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "fjw" + allRows << data + } + return allRows; + } + + def findCityInfo(city){ + def q = new BasicDBObject() + q.put("display", city) + return CcbCommonUtils.SERVER_AREA_COLL.findOne(q) + } + + def isCapital(def prov, def city){ + def provCity = CcbCommonUtils.provCityMap.get(prov) + if(provCity && provCity==city){ + return "是" + } + return "否" + } + + def ifMunicipal(def city){ + if(["上海", "北京", "天津", "重庆"].contains(city)){ + return "是" + } + return "否" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy new file mode 100644 index 000000000..be208aebb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql + +/** + * 建行模型估值项目 + * 估价系数数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class EvaluateRateBatch extends BaseBatch{ + + static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver") + + static main(args) { + runBatch(new EvaluateRateBatch(), args); + } + + @Override + public void process(Object args) { + + def allDatas = createDatas(findEvaluationRate()) + def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false); + } + + def createDatas(def datas){ + def allRows = [] + datas.each { m -> + def data = [] + data << m.id.toString() + data << m.rate_type + data << m.rate + data << m.rank + data << "全国" + data << m.rank_remark + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + allRows << data + } + return allRows + } + + private def findEvaluationRate(){ + def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'"; + def rets = [] + sqlInstance.eachRow(sql,{ + rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated] + }); + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy new file mode 100644 index 000000000..234d172b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 建行模型估值项目 + * 人工指导价同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class GuiDingPriceBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new GuiDingPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + + } + + def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false); + + def rets = [] + rets << headers + rets.addAll(rows) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") + + } + + def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){ + def rets = [] + if(prices){ + for(def ps : prices){ + def dist = distMap.get(ps.region+"##"+ps.name) + if(dist){ + def ret = [] + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + if(dist.region){ + ret << distCodeMap.get(dist.region) + }else{ + ret << "" + } + ret << dist.region + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << ps.guiding_value + ret << ps.guiding_person + ret << "" + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + } + return rets; + } + + def getDistrictMap(def city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + } + return distMap; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy new file mode 100644 index 000000000..4ecac813a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +/** + * 建行模型估值项目 + * 分期数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class StagingDistrictBatch extends BaseBatch{ + + static main(args) { + runBatch(new StagingDistrictBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"]) + def fenQiMap = getFenQiMap(dists); + def rets = createCityData(dists, fenQiMap) + rows.addAll(rets) + println city +" 分期量:"+rets.size() + } + + + def results =[] + def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + results << headers + results.addAll(rows) + + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据") + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8"); + + } + + def createCityData(def dists, def fenQiMap){ + def rets = [] + for(def d : dists){ + if(d.master_district_name && d.master_district_name!="master"){ + def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name) + if(masterDist){ + def ret = [] + ret << new ObjectId().toString() + ret << d._id.toString() + ret << masterDist._id.toString() + ret << "" + ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } +// else{ +// def udata = new BasicDBObject() +// udata.put("master_district_name", null) +// udata.put("m_date", new Date()) +// println d.city +" "+d.name+" "+d._id.toString() +// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata)) +// } + } + } + return rets; + } + + def getFenQiMap(def dists){ + def fenQiMap = [:] + for(def d : dists){ + if(d.master_district_name){ + fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d) + } + } + return fenQiMap; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy new file mode 100644 index 000000000..d4e252dd5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -0,0 +1,177 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 建行模型估值项目 + * 第三方均价数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class ThirdPartyPriceBatch extends BaseBatch { + + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + + static main(args) { + runBatch(new ThirdPartyPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + def index = 0 + for(def city : citys){ + index++ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"]) + def thirdPriceMap = findThirdPartyPrice(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) + rows.addAll(rets) + println index+" "+city+" "+rets.size() + } + def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false); + + } + + + def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def rets = [] + for(def d : dists){ + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def ret = [] + ret << third._id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + if(d.region){ + ret << distCodeMap.get(d.region) + }else{ + ret << "" + } + ret << d.region + ret << third.s_date + + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + + } + return rets; + } + + + def findThirdPartyPrice(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", curDate) + def dbcursor = AVERAGE_STATISTICS.find(q) + def map = [:] + for(def db : dbcursor){ + map.put(db.region+"##"+db.name, db) + } + return map + } + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy new file mode 100644 index 000000000..af92f0489 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy @@ -0,0 +1,174 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +class BestDistrictPriceExport { + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new BestDistrictPriceExport(); + task.run() + } + + def run(){ + def city = "佛山" + def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + def rets = [] + rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"] + def distMap = findCreditHightDistrict2(city, "20210711"); + distMap.each{region, dists-> + + if(region=="三水"){ + def addIndex = 0 + for(def dist : dists){ + if(addIndex <=30){ + def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name) + def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711") + if(otherPrice){ + def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price + if(Math.abs(precent)>=0 && Math.abs(precent)<=10){ + def ret = [] + ret << "佛山" + ret << dist.region + ret << dist.district_name + ret << dist.estate_type + for(def weekend : months){ + def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend) + if(obj){ + ret << obj.avg_price + }else{ + ret << "" + } + } + ret << otherPrice.lianjia_esf_price + ret << stat.credit + rets << ret + addIndex++ + } + } + } + } + } + } + ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格") + } + + + def findDistrictPrice(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def obj = DIST_SELL_STAT.findOne(q) + return obj + } + + def findCreditHightDistrict(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def typeMap = retMaps.get(d.region) + if(!typeMap){ + typeMap = [:] + retMaps.put(d.region, typeMap) + } + + def rets = typeMap.get(d.estate_type) + if(!rets){ + rets = [] + typeMap.put(d.estate_type, rets) + } + + def count = 0 + typeMap.each {k,v-> + if(v){ + count = count + v.size() + } + } + + if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){ + def m = [:] + m.city = d.city + m.region = d.region + m.district_name = d.district_name + rets << m + }else if(rets.size() <= 20){ + def m = [:] + m.city = d.city + m.region = d.region + m.district_name = d.district_name + rets << m + } + + def total = 0 + retMaps.each {k,m-> + m.each {t,v-> + total = total + v.size() + } + } + if(total>100){ + break; + } + } + + def rows = [] + retMaps.each {k,m-> + m.each {t,v-> + rows.addAll(v) + } + } + return rows + } + + + def findCreditHightDistrict2(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + def m = [:] + m.city = d.city + m.region = d.region + m.estate_type = d.estate_type + m.district_name = d.district_name + rets << m + } +// +// def rows = [] +// retMaps.each {k,v-> +// rows.addAll(v) +// } + return retMaps; + } + + def findNearThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", 202106) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + return AVERAGE_STATISTICS.findOne(q) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy new file mode 100644 index 000000000..4687872d3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy @@ -0,0 +1,265 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.apache.lucene.queryparser.classic.QueryParser + +class SourceDistDataMatch { + + static main(args) { + def task = new SourceDistDataMatch(); +// task.run() + task.mapRun() +// task.filterName() +// task.filterType() + } + + def run(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(obj){ + d['R'] = 1 + }else{ + def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709) + if(distCount){ + d['W'] = distCount + } + } + },0) + } + + def mapRun(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N'] && !d['F']){ + def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']] + def locationMap = matchMap(dist) + if(locationMap){ + d['F'] = locationMap.source + d['G'] = locationMap.region + d['H'] = locationMap.mapName + d['I'] = locationMap.address + d['J'] = locationMap.type + d['K'] = locationMap.lng + d['L'] = locationMap.lat + d['M'] = locationMap.score + } + } + },0) + } + + def filterName(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N']) { + def name = d['C'] + if (name) { + def flag = NameProcess.filterName(name) + if (!flag) { + d['N'] = 4 + } + } + } + },0) + + } + + def filterType(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N']){ + def type = d['J'] + if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){ + + }else if(type && type.contains("商务住宅") && !type.contains("商住两用")){ + def count = 0 + if(d['S'] && d['S']!='null'){ + count = d['S'].toDouble().toInteger() + } + if(count<10){ + d['N'] = 3 + } + }else{ + if(type){ + d['N'] = 3 + } + } + } + },0) + } + + + def matchMap(def dist){ + def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") + if(!gaodeRest && dist.address){ + gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德") + } + def gdlocationMap = null + def bdlocationMap = null + if(gaodeRest){ + gdlocationMap = getMapLocation2(dist, gaodeRest); + if(gdlocationMap){ + gdlocationMap.source = "高德" + } + } + + def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + if(baiduRest){ + bdlocationMap = getMapLocation2(dist, baiduRest); + if(bdlocationMap){ + bdlocationMap.source = "百度" + } + } + + if(gdlocationMap && bdlocationMap){ + if(gdlocationMap.score>=bdlocationMap.score){ + return gdlocationMap + }else{ + return bdlocationMap + } + }else if(gdlocationMap){ + return gdlocationMap + }else if(bdlocationMap){ + return bdlocationMap + } + return null + } + + + + def findHouseCount(def city, def region, def alias, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } + + def getMapLocation2(def dist, def rets){ + def data1 = findSimilarData(dist, rets, true) + def data2 = findSimilarData(dist, rets, false) + println data1 + println data2 + if(data1 && data1.score>1){ + return data1 + }else if(data2 && data2.score>1){ + return data2 + } + return null + } + + def findSimilarData(def dist, def rets, def regionImpr){ + def lat = null + def lng = null + def region = null + def mapName = null + def address = null + def score = 0 + def type = null + def maxScore = 0 + def standardName = null + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + if(["公交车站"].contains(r.type)){ + continue + } + if(regionImpr && !r.district){ + continue + } + if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){ + continue + } + + def name = nameReplace(r.name, dist.city) + + def distName = nameReplace(dist.name, dist.city) + + def score3 = 0.0 + if(r.district && dist.region && r.district.contains(dist.region)){ + score3 = 0.2 + } + + def score1 = 0.0; + def score2 = 0.0 + if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){ + if(r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2 + } + }else{ + score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2; + if(dist.address && r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2 + } + } + + score3 = score1 + score2 + if(r.type && r.type.contains("住宅")){ + score3 = score3 + 0.2 + } + if(score3 >maxScore){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + standardName = name + address = r.address + type = r.type + score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10) + maxScore = score3 + } + println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3 + + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName] + } + return null + } + + def nameReplace(def s, city){ + if(s.contains("(") && s.contains(")")){ + def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")")) + if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){ + s = s.toString().substring(0, s.indexOf("(")) + } + } + def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim() + if(name.startsWith(city)){ + def newName = name.replace(city+"市","").replace(city,"") + if(newName && newName=="花园"){ + return name + }else{ + name = newName + } + } + return name + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy new file mode 100644 index 000000000..40748a9fd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.gj.region + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class MapAreaUpdate { + static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + static main(args) { + def task = new MapAreaUpdate() + task.run() + } + + def run(){ + def rets = findAllCity() + for(def city : rets){ + def retMap = findCityAndProvCode(city) + if(retMap){ + update(retMap) + } + } + } + + def update(retMap){ + def q = new BasicDBObject() + q.put("fangjia_city",retMap.fangjia_city) + q.put("district", "市辖区") + def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q) + if(obj){ + def udata = new BasicDBObject() + udata.put("city_code", retMap.city_code) + udata.put("province_code", retMap.province_code) + FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false) + } + } + + def findCityAndProvCode(def city){ + def q = new BasicDBObject() + q.put("fangjia_city",city) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def map = [:] + for(def d : dbcursor){ + if(d.city_code && d.province_code && d.district_code){ + map.put("fangjia_city",d.fangjia_city) + map.put("city_code",d.city_code) + map.put("province_code",d.province_code) + break + } + } + return map; + } + + def findAllCity(){ + def q = new BasicDBObject() + q.put("fangjia_district",null) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def citys = [] + for(def d : dbcursor){ + citys << d.fangjia_city + } + citys.unique() + return citys + } +} From ed0f3a05d413828ebaf0a553c84add80e610e1bd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 22 Jul 2021 09:59:18 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=8A=93=E6=97=A0=E9=94=A1=E6=9D=91?= =?UTF-8?q?=E5=BA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/core/util/BaiDuApiUtils.java | 3 +- .../BigdataCommbuildtypeBatch.groovy | 88 +++ .../BigdataCommunityBatch.groovy | 128 +++-- .../ccbevaluation/BigdataDealBatch.groovy | 416 ++++++++++++++ .../ccbevaluation/BigdataDistrictBatch.groovy | 6 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 11 +- .../ccbevaluation/BigdataForsaleBatch2.groovy | 521 ------------------ .../ccbevaluation/CcbCommonUtils.groovy | 51 +- .../script/ccbevaluation/CityBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 21 + .../script/gj/AddThirdPriceDistrict.groovy | 59 +- .../fangjia/script/gj/GaoDeMapCrawl.groovy | 11 +- .../script/gj/data/ExportDistrictInfo.groovy | 78 +++ 13 files changed, 772 insertions(+), 623 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index e075fbd32..582a71f23 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -258,8 +258,9 @@ public class BaiDuApiUtils { params.put("coords", lng.toString() + "," + lat.toString()); params.put("from", String.valueOf(from)); HttpRequest request = new HttpRequest(); + String result = null; try{ - String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); + result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); JSONObject json = JSONObject.fromObject(result); while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){ changeAK(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy new file mode 100644 index 000000000..93864962e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -0,0 +1,88 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 建行模型估值项目 + * 小区建筑类型数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataCommbuildtypeBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new BigdataCommbuildtypeBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rets = [] + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"]) + def rows = createDatas(cityCodeDatas, distCodeMap, dists) + rets.addAll(rows) + println city +" "+rows.size() + } + def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false); + println "txt 生成" + + } + + def createDatas(cityCodeDatas, distCodeMap, dists){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i> FITMENTS_TYPES_MAPS = new HashMap>() - static{ - FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) - FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) - FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) - FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) - } - - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) - - def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] - - static main(args) { - runBatch(new BigdataForsaleBatch2(), args); - } - - @Override - public void process(Object args) { - def citys = CcbCommonUtils.citys; - for(def city : citys){ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def distMap = getDistrictMap(city) - exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731); - } - } - - def getDistrictMap(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); - def distMap = [:] - for(def d : districts){ - distMap.put(d.region+"##"+d.name, d) - if(d.alias){ - d.alias.each{name-> - if(!distMap.containsKey(d.region+"##"+name)){ - distMap.put(d.region+"##"+name, d) - } - } - } - } - return distMap; - } - - def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){ - println city +" 小区key:"+distMap.size() - def cityCode = cityCodeDatas.city_code - def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap) - def allDatas = createSellDatas(city, retMap); - def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true); - println "txt文件生成 " - } - - - - def createSellDatas(def city, def map){ - - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(map.size()) - println city +"总房源量:"+map.size() - - def datas = [] - map.each{k,m-> - fixedThreadPool.execute(new Runnable() { - void run() { - try { - totalCount.addAndGet(1) - datas << converData(m) - println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString() - } catch (Exception e) { - e.printStackTrace() - } finally { - latch.countDown() - } - }; - }) - } - latch.await() - return datas ; - } - - def converData(def m){ - def data = [] - data << m.city_code - data << m.houseId - data << "" - data << m.city - data << m.region_code - data << m.region - data << m.district_id - data << m.district_name - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << m.address==null?"":m.address==null - data << m.build_year==null?"":m.build_year - data << m.proptype==null?"":m.proptype - data << m.propyears==null?"":m.propyears - data << m.avg_price==null?"":m.avg_price - data << m.total_price==null?"":m.total_price - data << "" - data << m.floor==null?"":m.floor - data << m.height==null?"":m.height - data << m.estate_type==null?"":m.estate_type - data << m.house_structure_code==null?"":m.house_structure_code - data << m.build_structure_code==null?"":m.build_structure_code - data << m.area==null?"":m.area - data << "" - data << m.house_type==null?"":m.house_type - data << m.room==null?"":m.room - data << m.hall==null?"":m.hall - data << "" - data << m.toilet==null?"":m.toilet - data << m.direction_code==null?"":m.direction_code - data << m.fitment_code==null?"":m.fitment_code - data << "" - data << "" - data << m.p_date==null?"":m.p_date - data << "" - data << "" - data << m.root - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << m.c_date==null?"":m.c_date - data << m.m_date==null?"":m.m_date - data << m.source - data << "" - return data - } - - - def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){ - - QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); - QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); - - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) - .must(date_range).must(area_range) - - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); - def retMap = new ConcurrentHashMap() - println city +" 没过滤前:"+ houses.size() - if(houses){ - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(houses.size()) - - for (int k=0;k60){ - return null - } - -// if(!seawater.room){ -// continue; -// } - if(!seawater.room){ - seawater.room = null - } - - if(seawater.room){ - if(seawater.room>=5 || seawater.room<1){ - return null - } - } - - if(!seawater.hall){ - seawater.hall = null - } - if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ - return null - } - if(!seawater.toilet){ - seawater.toilet = null - } - if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ - return null - } - - - def floor = parseFloor(seawater.floor_info) - if(seawater.height && floor && floor.isNumber()){ - if(floor.toInteger()>seawater.height){ - floor = null - seawater.height = null - } - - } - - if(!floor){ - seawater.floor = null - } - if(floor && floor.toInteger()<1){ - floor = null - } - - def rowMap = [:]; - if(cat == "rent"){ - rowMap.rent_type = seawater.rent_type - if(seawater.avg_price && seawater.avg_price>1000){ - return null - } - - if(seawater.avg_price && seawater.avg_price<10){ - return null - } - - } - rowMap.houseId = seawater.id - rowMap.city = city - //rowMap.region =seawater.region - //rowMap.address = seawater.address - - - rowMap.area = seawater.area - rowMap.total_price = seawater.total_price - rowMap.avg_price = seawater.avg_price - rowMap.height = seawater.height=="null"?null:seawater.height - if(rowMap.height && rowMap.height<1){ - rowMap.height = null - } - rowMap.floor = floor=="null"?null:floor - rowMap.room = seawater.room=="null"?null:seawater.room - rowMap.hall = seawater.hall=="null"?null:seawater.hall - rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet - if(rootMap.get(seawater.root)){ - rowMap.root = rootMap.get(seawater.root) - }else{ - rowMap.root = "99" - } - - def direction = parseDirection(seawater.direction) - rowMap.direction = direction=="null"?null:direction - - def fitment = parseFitment(seawater.fitment) - rowMap.fitment = fitment=="null"?null:fitment - - - - if(seawater.p_date instanceof String){ - seawater.p_date = seawater.p_date.toInteger() - } - if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ - rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") - rowMap.c_date = rowMap.p_date - rowMap.m_date = rowMap.p_date - }else{ - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) - } - - def regionCode = distCodeMap.get(rMap.region) - def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) - def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) - def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) - def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type); - def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); - rowMap.city_code = cityCode - rowMap.region = rMap.region - rowMap.region_code = regionCode - rowMap.district_id = rMap.get("_id").toString() - rowMap.district_name = rMap.name - rowMap.build_year = buildYear; - rowMap.proptype = housePro - rowMap.propyears = propertyYear; - rowMap.estate_type = estateType - rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall); - rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction) - rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment); - rowMap.address = rMap.address - rowMap.build_structure_code = struCode - rowMap.house_structure_code = null//getHouseStructureCode(); - rowMap.source = "FJW" - return rowMap - - } - - def getPublicDate(def pDate){ - - if(pDate>=20181001 && pDate<20200201){ - return DateUtils.getRandomDate("2018-10-01", "2020-02-01") - } - def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") - return pdate; - } - - static def parseFloor(def floorDesc){ - if(floorDesc){ - floorDesc = floorDesc.replaceAll("[\\s ]+", "") - if(floorDesc){ - Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); - Matcher matcher1 = pat1.matcher(floorDesc); - if (matcher1.find()) { - return matcher1.group(1) - } - pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); - matcher1 = pat1.matcher(floorDesc); - if (matcher1.find()) { - return matcher1.group(1) - } -// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ -// return "中层" -// } -// -// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ -// return "高层" -// } -// -// -// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ -// return "低层" -// } - if(floorDesc.isNumber()){ - return floorDesc; - } - } - } - return null; - } - - private String csvHandlerStr(String str) { - //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 - String tempDescription=str; - //如果有逗号 - if(str.contains(",")||str.contains(",")){ - //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 - if(str.contains("\"")){ -// tempDescription=str.replace("\"", "\"\""); - tempDescription=str.replace("\"", ""); - } - //在将逗号转义 - tempDescription="\""+tempDescription+"\""; - } - return tempDescription; - } - - - static String parseFitment(String fitmentDesc) { - if (fitmentDesc == null) { - return null - } - fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") - if (fitmentDesc.equals("非毛坯")) { - return null - } - Iterator>> it = FITMENTS_TYPES_MAPS - .entrySet().iterator() - Map.Entry> entry - while (it.hasNext()) { - entry = it.next() - if (fitmentDesc.contains(entry.getKey())) { - return entry.getKey() - } - for (String fitmentKey : entry.getValue()) { - if (fitmentDesc.contains(fitmentKey)) { - return entry.getKey() - } - } - } - return null - - } - - static String parseDirection(String directionDesc) { - if (directionDesc == null) { - return null - } - directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") - StringBuffer sb = new StringBuffer() - for (int i = 0; i < directionDesc.length(); i++) { - String c = directionDesc.substring(i, i + 1) - if (DIRECTIONS.contains(c)) { - if (directionDesc.substring(i).startsWith(c + "#_#")) { - sb.append(c + " ") - } else { - sb.append(c) - } - } - } - for (String d : DIRECTIONS) { - if (sb.toString().startsWith(d)) { - return d - } - } - for (String d : DIRECTIONS) { - if (sb.toString().contains(d)) { - return d - } - } - return null - - } - - /* - * 生成一个用不重复的ID - */ - def GenNonDuplicateID(randomLength){ - return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) - } - - - - def marge(){ - File file = new File("D:\\数据\\建行\\新建文本文档.txt"); - def ids = new HashSet(); - def index = 0 - - file.eachLine{it-> - index++ - if(index>1){ - def arrs = it.split("\\|@\\|") - def id = arrs[1] - if(!ids.contains(id)){ - ids << id - } - } - } - println index - println ids.size() - } - -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 37c99bc96..c11a06cfa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -7,8 +7,8 @@ import com.mongodb.DBCollection class CcbCommonUtils { - public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] -// public static def citys = ["北京"] + public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] +// public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -18,6 +18,11 @@ class CcbCommonUtils { public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static def findCityCodeValue(city){ def q = new BasicDBObject() q.put("fangjia_city", city) @@ -58,4 +63,46 @@ class CcbCommonUtils { } return rets } + + public static def findTradesDetail(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + def lists = [] + for(def d : dbcursor){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } + } + return lists + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy index bafa9e4d8..ef52b58c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -52,7 +52,7 @@ class CityBatch extends BaseBatch{ rets << headers rets.addAll(allDatas) ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8"); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy new file mode 100644 index 000000000..399e3c53a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -0,0 +1,21 @@ +package com.fangjia.script.ccbevaluation +import groovy.sql.Sql + +def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +def citys = CcbCommonUtils.citys; +def rows = [] + +for(def city : citys) { + def data = CcbCommonUtils.findCityCodeValue(city) + def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; + + def count = 0 + sqlInstance.execute(sql,{ + count = it.count + }); + println city +" "+count +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy index aece3ef15..d61bfb2cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy @@ -74,20 +74,20 @@ class AddThirdPriceDistrict { static main(args) { def task = new AddThirdPriceDistrict(); -// task.run() + task.run() - def mapName = "锦华星光苑3,4幢" +// def mapName = "锦华星光苑3,4幢" //// println task.pattern("港新花苑A1区"); - println mapName.replaceAll("\\w+-?\\w+(幢)", "") - println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") +// println mapName.replaceAll("\\w+-?\\w+(幢)", "") +// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") } def run(){ -// exportDatas(); + exportDatas(); // getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); // addInfo(); // addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx") // addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx") - addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); +// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); } def addCrawlMapInfo( def path){ @@ -135,29 +135,32 @@ class AddThirdPriceDistrict { def exportDatas(){ - def city = "无锡" - def datas = findMatchErrorDatas(city); - println "共【"+datas.size()+"】条" - datas = filterName(datas); - println "过滤后共【"+datas.size()+"】条" - def resMap = groupByName(datas); - def rets = [] - rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"] - resMap.each {k,v-> - def ret = [] - def arrs = k.split("##") - ret << arrs[0] - ret << arrs[1] - if(arrs.size()==3 && arrs[2]){ - ret << arrs[2] - }else{ - ret << "" + //跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞", + def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + for(def city : citys){ + def datas = findMatchErrorDatas(city); + println "共【"+datas.size()+"】条" + datas = filterName(datas); + println "过滤后共【"+datas.size()+"】条" + def resMap = groupByName(datas); + def rets = [] + rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"] + resMap.each {k,v-> + def ret = [] + def arrs = k.split("##") + ret << arrs[0] + ret << arrs[1] + if(arrs.size()==3 && arrs[2]){ + ret << arrs[2] + }else{ + ret << "" + } + ret << resAddrMap.get(k) + ret << v.join(",") + rets << ret } - ret << resAddrMap.get(k) - ret << v.join(",") - rets << ret + ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后") } - ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后") } def filterName(datas){ @@ -465,7 +468,7 @@ class AddThirdPriceDistrict { def findMatchErrorDatas(def city){ def q = new BasicDBObject() q.put("city",city) - q.put("s_date",202005) + q.put("s_date",new BasicDBObject('$gte', 202101)) // q.put("name","博济聚创科技园") // q.put("source","贝壳找房") def dbcursor = matchErrorLogColl.find(q) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 00c1ec513..a8b29aff0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -56,11 +56,11 @@ class GaoDeMapCrawl extends BaseBatch{ // def prefix = "road_" // def categorys = ["190301"]//道路相关 -// def prefix = "village_" -// def categorys = ["190108","190109"]//村庄 + def prefix = "village_" + def categorys = ["190108","190109"]//村庄 - def prefix = "company_" - def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] +// def prefix = "company_" +// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] def totalCount = new AtomicLong() static main(args) { @@ -118,7 +118,8 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" - def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys =["无锡"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy new file mode 100644 index 000000000..747ddf078 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + + +def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"] + +def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"] +def rets = [] +rets << headers + +for(def s : districts){ + def arrs = s.split("_") + def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2]) + def ret = [] + ret << s + if(m){ + ret << m.city + ret << m.district + ret << m.name + if(m.alias){ + ret << m.alias.toString().replace("[","").replace("]","") + }else{ + ret << "" + } + ret << m.address + if(m.addressAlias){ + ret << m.addressAlias.toString().replace("[","").replace("]","") + }else{ + ret << "" + } + + ret << m.avgPrice + ret << m.lng + ret << m.lat + ret << m.block + ret << m.loopLine + ret << m.propertyType + ret << m.buildingType + ret << m.commercialArea + ret << m.totalUnitCount + ret << m.managementFee + ret << m.parking + ret << m.parkingDescription + ret << m.ownershipPeriod + ret << m.ownershipType + ret << m.startBuildTime + ret << m.startSellingTime + ret << m.finishBuildTime + ret << m.moveinTime + ret << m.floorSpace + ret << m.constructionSpace + ret << m.plotRatio + ret << m.greenPercentage + ret << m.preSalePermit + ret << m.developer + ret << m.propertyManagement + ret << m.decorationStandard + ret << m.buildingStructure + ret << m.supportingFacilities + ret << m.transportation + ret << m.relatedInfo + ret << m.description + ret << m.salesOfficeAdd + ret << m.salesPhoneNumber + ret << m.discount + ret << m.buildingCount + ret << m.finishYear + + } + rets << ret +} +ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据") + + + + From be891b3229184cb5626b7194a3cf5335e9b96fe0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 22 Jul 2021 10:01:40 +0800 Subject: [PATCH 3/3] update --- .../fangjia/script/gj/EsSearchUtils.groovy | 2 +- .../com/fangjia/script/gj/FiledStrand.groovy | 1 + .../com/fangjia/script/gj/OpenApiUtils.groovy | 2 + .../fangjia/script/gj/WuXiAddressTest.groovy | 37 +++++++++++- .../fangjia/script/gj/YouDaDataParser.groovy | 6 +- .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 60 +++++++++++++++++++ .../fangjia/script/gj/ccb/FileUtils.groovy | 30 +++++----- .../script/gj/data/RegionPriceExport.groovy | 24 ++++++-- .../script/gj/district/ExportData.groovy | 46 +++++++------- .../gj/district/FangjiaClientUtils.groovy | 5 +- .../gj/district/MapDistrictProcess.groovy | 31 ++++++---- .../district/SeaweedDataMatchProcess.groovy | 13 ++-- .../script/gj/district/ZsPersonProcess.groovy | 2 +- .../script/gj/es/EsMonthHouseStat.groovy | 54 ++++++++--------- .../thirdPriceCrawl/ZhugefangCrawl4.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 1 + 16 files changed, 220 insertions(+), 96 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index dc7821343..4d50b3661 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -7,7 +7,7 @@ import java.sql.ResultSet class EsSearchUtils { public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1f5247429..3fd9abe32 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -394,6 +394,7 @@ class FiledStrand { } return estateCharege; } + static def processHouseholdCount(def household_count, def estateType2){ if(household_count){ household_count = household_count.toInteger() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 4ec26d7db..b5efa4f51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -29,6 +29,8 @@ class OpenApiUtils { rMap.region = dto.district rMap.name = dto.name rMap.id = dto.id + rMap.lng = dto.lng + rMap.lat = dto.lat if(dto.address && dto.address!="null"){ rMap.address = dto.address } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy index f8c8c0d5c..cbf9f5e12 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy @@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils class WuXiAddressTest { static main(args) { def task = new WuXiAddressTest() - task.run() + //task.run() + task.addAddress() + } + + def addAddress(){ + ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d-> + def prov = d['C'] + def city = d['D'] + def region = d['E'] + def address = d['B'] + def road = d['J'] + def street = d['R'] + + def newAddress = prov+"省"+city +"市" + if(region == "惠山"){ + newAddress = newAddress + region +"区" + }else{ + newAddress = newAddress + region +"市" + } + + d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "") + def dist = d['U'] + + def s1 = null + if(road){ + s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","") + newAddress = newAddress + street + dist + road + s1 + }else if(d['L']){ + s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","") + newAddress = newAddress + street + dist + d['L'] + s1 + } + + d['V'] = newAddress + },0) } def run(){ @@ -18,6 +51,8 @@ class WuXiAddressTest { // // } + + ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d-> if(!d['S']){ def locationStr = getLocationInfo(d['D'], d['Q']) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 0c2be14c8..d8ccfa02e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,11 +47,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-05-01" + static def statDate = "2021-06-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 89dff40d4..7a766b668 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -7,6 +7,13 @@ import java.util.regex.Pattern class CcbCodeUtils { + //否:0 是:1 + static def parseFlag(String flag){ + if(flag && flag == "是"){ + return 1; + } + return 0; + } static def parseBuildYear(String buildDate) { if(buildDate){ @@ -53,6 +60,44 @@ class CcbCodeUtils { return null } +// 高层塔楼 0101 增加“超高层” 0108 +// 高层板楼 0102 增加“老洋房” 0109 +// 小高层板楼 0103 增加“花园洋房” 0110 +// 多层板楼 0104 增加“四合院” 0111 +// 独栋别墅 0105 增加“老式别墅洋房” 0112 +// 联排别墅 0106 +// 叠拼别墅 0107 +// 其他建筑形式 0199 + + static def parseBuildType(def buildType){ + if(buildType){ + if(buildType.contains("老式别墅洋房")){ + return "0112" + }else if(buildType.contains("四合院")){ + return "0111" + }else if(buildType.contains("花园洋房")){ + return "0110" + }else if(buildType.contains("老洋房")){ + return "0109" + }else if(buildType.contains("叠拼")){ + return "0107" + }else if(buildType.contains("联排")){ + return "0106" + }else if(buildType.contains("独栋")){ + return "0105" + }else if(buildType.contains("多层")){ + return "0104" + }else if(buildType.contains("超高层")){ + return "0108" + }else if(buildType.contains("小高层")){ + return "0103" + }else if(buildType.contains("高层")){ + return "0102" + } + } + return "0199" + } + //朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399 static def getDirectionCode(def direction){ @@ -161,4 +206,19 @@ class CcbCodeUtils { } return 0; } + + + //房屋用途 + static def parseHouseUse(String houseUse){ + if(houseUse){ + if(houseUse.contains("住宅")){ + return 11; + }else if(houseUse.contains("商住")){ + return 13; + }else{ + return 99 + } + } + return 99 + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 09f80b9da..9382cb398 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -3,13 +3,13 @@ package com.fangjia.script.gj.ccb import org.apache.commons.lang.StringUtils class FileUtils { - static def batchCreateFile(ret, fileName, headers,charset){ + static def batchCreateFile(ret, fileName, headers,charset, isAppend){ File file = new File(fileName); - if (file.exists()) { - file.delete(); - } +// if (file.exists()) { +// file.delete(); +// } - OutputStream out = new FileOutputStream(fileName); + OutputStream out = new FileOutputStream(fileName, isAppend); def sb = new StringBuffer(); sb.append(headers.join("|@|")) @@ -20,18 +20,18 @@ class FileUtils { def index = 0 for(def row : ret){ - index++ - sb.append(StringUtils.join(row.toArray(), "|@|")) - sb.append(System.getProperty("line.separator")); - println index + if(row){ + index++ + sb.append(StringUtils.join(row.toArray(), "|@|")) + sb.append(System.getProperty("line.separator")); - if(index%10000==0){ - println index+"开始写入CSV "+new Date().toLocaleString() - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - println index+"结束写入CSV "+new Date().toLocaleString() + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } } - } if(sb.length()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy index d059de4ae..a56e2079d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.data import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -10,22 +11,23 @@ import com.mongodb.DBCollection class RegionPriceExport { DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com") + DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") static main(args) { def task = new RegionPriceExport(); task.run() } def run(){ - def months = [202010,202011,202012,202101,202102,202103] - def regions = CommonUtils.getAllRegions("福州","region") + def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + def regions = CommonUtils.getAllRegions("佛山","region") def rets = [] - rets << ["城市","区域","202010","202011","202012","202101","202102","202103"] + rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] regions.each{region,values-> def data = [] - data << "福州" + data << "佛山" data << region for(def month :months){ - def dbo = findPrice("福州", region, month); + def dbo = findWeekendPrice("佛山", region, month); if(dbo){ data << dbo.avg_price }else{ @@ -34,7 +36,7 @@ class RegionPriceExport { } rets << data } - ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格") + ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格") } def findPrice(city, region, month){ @@ -45,4 +47,14 @@ class RegionPriceExport { def dbo = REGION_SELL_STAT.findOne(q) return dbo } + + def findWeekendPrice(city, region, month){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("cat", "区域") + q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) + def dbo = REGION_STAT.findOne(q) + return dbo + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index 6a44d4139..6f9219576 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.gj.district import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject @@ -13,7 +15,7 @@ class ExportData { def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol(); public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -24,8 +26,8 @@ class ExportData { static main(args) { def task = new ExportData(); - task.exportThird() -// task.exportMap(); +// task.exportThird() + task.exportMap(); } def exportMap(){ @@ -51,19 +53,18 @@ class ExportData { //"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特" //宁夏 ----- 中卫 固原 石嘴山 吴忠 银川 //青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁 - def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州", - "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮", - "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"] - //"沈阳","大连","抚顺","鞍山" - //"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆" - //"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛", - //"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州", - //"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城", +// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州", +// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮", +// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"] + def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]; + + for(def city : citys){ + def path = "D:\\数据\\小区\\地图\\${city}.xlsx"; + def datas = findAllDatas(city, null) + println "${city}共计:"+datas.size() + ExcelUtils.write(path, datas, "数据") + } - def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx"; - def datas = findAllDatas(citys, null) - println "共计:"+datas.size() - ExcelUtils.write(path, datas, "数据") } @@ -98,9 +99,9 @@ class ExportData { - def findAllDatas(def citys, def pname){ + def findAllDatas(def city, def pname){ final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1") + sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'") // sqlBuffer.append(" and pname = '"+pname+"'") // sqlBuffer.append(" and cityname = '"+pname+"'") ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) @@ -113,16 +114,19 @@ class ExportData { if(source && source=="高德"){ println index def type = rs.getObject("type") - def city = rs.getString("cityname") - if(citys.contains(city) && type.contains("商务住宅")){ + //def city = rs.getString("cityname") + if(type.contains("住宅")){ def ret = [] ret << city ret << rs.getString("adname") ret << rs.getString("name") ret << rs.getString("address") ret << rs.getString("type") - ret << rs.getFloat("location.lon") - ret << rs.getFloat("location.lat") + def loaction = rs.getObject("location") + if(loaction){ + ret << DataHandleUtils.getDoubleRound(6, loaction.x) + ret << DataHandleUtils.getDoubleRound(6, loaction.y) + } rets << ret } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 8eacf492b..ac634e9e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -68,13 +68,16 @@ class FangjiaClientUtils { public static def addressStand(def address){ def jsonObj = null + def content = null try { + address = address.replace("{","") def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") - def content = new HttpRequest().newGet(url, null, "utf-8") + content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) } catch (Exception e) { + println address +" "+content e.printStackTrace(); } return jsonObj; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy index 8f5809bed..f53276b72 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy @@ -1,9 +1,12 @@ package com.fangjia.script.gj.district +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DataUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation import java.util.regex.Pattern @@ -20,20 +23,13 @@ class MapDistrictProcess { * @return */ def run(){ - //"江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx","安徽地图.xlsx", - //"上海地图.xlsx","北京地图.xlsx" -//"安徽地图.xlsx","江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx", - //"重庆地图.xlsx","天津地图.xlsx", - //"上海地图.xlsx","北京地图.xlsx", - //"河北地图.xlsx","河南地图.xlsx","湖北地图.xlsx","湖南地图.xlsx","广西地图.xlsx", - //"吉林地图.xlsx","辽宁地图.xlsx""辽宁地图.xlsx","吉林地图.xlsx","黑龙江地图.xlsx","山西地图.xlsx","海南地图.xlsx"] - def files = ["青海地图.xlsx","宁夏地图.xlsx","内蒙古地图.xlsx","云南地图.xlsx","陕西地图.xlsx","新疆地图.xlsx","四川地图.xlsx"] + def files = ["武汉.xlsx"] for(def fileName : files){ def filePath = "D:\\数据\\小区\\地图\\"+fileName; - typeFilter(filePath) - nameStandard(filePath); - dataFilter(filePath) +// typeFilter(filePath) +// nameStandard(filePath); +// dataFilter(filePath) matchDistDatas(filePath) houseCalculate(filePath); ////// //统计是否存在多个区域 @@ -114,15 +110,24 @@ class MapDistrictProcess { def matchDistDatas(def path){ def index = 0 - ExcelUtils.multiThreadingModify(path,{ List row -> + ExcelUtils.modify(path,{ List row -> index ++ if(!row['I'] && !row['J']){ - def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],); + def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']); if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){ row['J'] = rMap.city row['K'] = rMap.region row['L'] = rMap.name row['M'] = rMap.address + + if(rMap.lat){ + def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3) + def maplng = baidu.x + def maplat = baidu.y + row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng) + def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L']) + row['Y'] = score + } } println index +" "+row } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 6a5c21a8e..5b0910fc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -26,15 +26,18 @@ class SeaweedDataMatchProcess { } } - if(!rMap && address){ -// if (findPattern(address)){ - rMap = OpenApiUtils.matchDistrict(city, address, region, true) -// } - } if(!rMap && name){ rMap = OpenApiUtils.matchDistrict(city, name, region, false) } + if(!rMap && address){ + rMap = OpenApiUtils.matchDistrict(city, address, region, true) + if(rMap && rMap.region && !region.contains(rMap.region)){ + rMap = null + } + } + + if(rMap && rMap.id && region.contains(rMap.region)) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy index e760b169f..cfa92f44a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy @@ -49,7 +49,7 @@ class ZsPersonProcess { // def loactionMap = task.matchMap(dist) // println loactionMap -// task.matchMap() + task.matchMap() // task.matchDb() // task.matchDb2() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index d8c7e72a3..be10cf533 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,9 +19,9 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() -// task.houseRun() + task.houseRun() - task.statHouseUpdateCount() +// task.statHouseUpdateCount() } def run(){ @@ -99,37 +99,35 @@ class EsMonthHouseStat { } def houseRun(){ - ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d-> - def city = d['B'] - if(city == "湖州"){ - def count = findHouseCount(city, 20210301, 20210601) - d['E'] = count + ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d-> + def city = d['A'] + def count = findHouseCount(city, 20190709, 20210709) + d['E'] = count - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); - def disthouseCount = 0 - for(def dist : districts){ - def alias = [] - if(dist.alias){ - if(!dist.alias.contains(dist.name)){ - dist.alias << dist.name - } - alias = dist.alias - }else{ - alias << dist.name + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def disthouseCount = 0 + for(def dist : districts){ + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name } - - def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601) - if(distCount){ - disthouseCount++ - } - println dist.city+" "+dist.region+" "+dist.name+" "+distCount + alias = dist.alias + }else{ + alias << dist.name } - println "disthouseCount:"+disthouseCount - println districts.size() - def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) - d['D'] = precent+"%" + def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709) + if(distCount){ + disthouseCount++ + } + println dist.city+" "+dist.region+" "+dist.name+" "+distCount } + + println "disthouseCount:"+disthouseCount + println districts.size() + def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) + d['F'] = precent+"%" },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy index c697657e8..a57c51cc7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy @@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{ @Override void process(Object args) { - citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""]) + citys.addAll(["福州"]) // getNoDataCitys(); def cityurl = "http://www.zhuge.com/areachange/" def cityUrls = getCityUrlCrawal(cityurl); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 6e5696107..841fb2f08 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -140,6 +140,7 @@ public class OpenApiClient { return null } + public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) { if (StringUtils.isBlank(input)) {