From 7dff8154c5a449b5f73f80fed61dfb6e93b60c96 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 17 Jul 2021 20:54:44 +0800 Subject: [PATCH] add ccb batch file --- .../BigdataCommunityBatch.groovy | 154 ++++++ .../ccbevaluation/BigdataDistrictBatch.groovy | 91 +++ .../ccbevaluation/BigdataForsaleBatch.groovy | 520 +++++++++++++++++ .../ccbevaluation/BigdataForsaleBatch2.groovy | 521 ++++++++++++++++++ .../ccbevaluation/CcbCommonUtils.groovy | 61 ++ .../script/ccbevaluation/CityBatch.groovy | 100 ++++ .../ccbevaluation/EvaluateRateBatch.groovy | 64 +++ .../ccbevaluation/GuiDingPriceBatch.groovy | 99 ++++ .../ccbevaluation/StagingDistrictBatch.groovy | 85 +++ .../ccbevaluation/ThirdPartyPriceBatch.groovy | 177 ++++++ .../gj/data/BestDistrictPriceExport.groovy | 174 ++++++ .../gj/district/SourceDistDataMatch.groovy | 265 +++++++++ .../script/gj/region/MapAreaUpdate.groovy | 65 +++ 13 files changed, 2376 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy new file mode 100644 index 000000000..3913caee7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -0,0 +1,154 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 建行模型估值项目 + * 小区数据同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class BigdataCommunityBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new BigdataCommunityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + + def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + "总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + "乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] + + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) + def rets = createCityDatas(cityCodeDatas, distCodeMap, dists) + rows.addAll(rets) + println city +" "+rets.size() + + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); + } + + def allDatas = [] + allDatas << headers + allDatas.addAll(rows) + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false); + println "txt 生成" + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") + } + + def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){ + def rets = [] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + def data = [] + data << m.dist_code + data << m.dist_name + data << m.dist_spell + data << m.city_id + data << m.dist_code + data << index + data << "" + data << "" + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + data << "" + data << "" + data << "" + data << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + index++ + allRows << data + } + return allRows; + } + + def findDistrictCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + q.put("district", new BasicDBObject('$ne',"市辖区")) + def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + def lists = [] + for(def db : dbcursor){ + if(db.district_code){ + def m = [:] + m.id = db._id.toString() + m.dist_name = db.fangjia_district + m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") + m.city_id = db.city_code + m.dist_code = db.district_code + lists << m + } + } + return lists; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy new file mode 100644 index 000000000..7e97743c6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -0,0 +1,520 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataForsaleBatch extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + + static main(args) { + runBatch(new BigdataForsaleBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731); + } + } + + def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){ + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println city +" 小区量:"+districts.size() + + def retMap = new ConcurrentHashMap() + for(def i=0;i0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = city + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(rootMap.get(seawater.root)){ + rowMap.root = rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + } + } + return rets; + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + /* + * 生成一个用不重复的ID + */ + def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) + } + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy new file mode 100644 index 000000000..c896476d1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch2.groovy @@ -0,0 +1,521 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataForsaleBatch2 extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + + def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + + static main(args) { + runBatch(new BigdataForsaleBatch2(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def distMap = getDistrictMap(city) + exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731); + } + } + + def getDistrictMap(def city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + if(d.alias){ + d.alias.each{name-> + if(!distMap.containsKey(d.region+"##"+name)){ + distMap.put(d.region+"##"+name, d) + } + } + } + } + return distMap; + } + + def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){ + println city +" 小区key:"+distMap.size() + def cityCode = cityCodeDatas.city_code + def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap) + def allDatas = createSellDatas(city, retMap); + def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true); + println "txt文件生成 " + } + + + + def createSellDatas(def city, def map){ + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(map.size()) + println city +"总房源量:"+map.size() + + def datas = [] + map.each{k,m-> + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + datas << converData(m) + println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return datas ; + } + + def converData(def m){ + def data = [] + data << m.city_code + data << m.houseId + data << "" + data << m.city + data << m.region_code + data << m.region + data << m.district_id + data << m.district_name + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << m.address==null?"":m.address==null + data << m.build_year==null?"":m.build_year + data << m.proptype==null?"":m.proptype + data << m.propyears==null?"":m.propyears + data << m.avg_price==null?"":m.avg_price + data << m.total_price==null?"":m.total_price + data << "" + data << m.floor==null?"":m.floor + data << m.height==null?"":m.height + data << m.estate_type==null?"":m.estate_type + data << m.house_structure_code==null?"":m.house_structure_code + data << m.build_structure_code==null?"":m.build_structure_code + data << m.area==null?"":m.area + data << "" + data << m.house_type==null?"":m.house_type + data << m.room==null?"":m.room + data << m.hall==null?"":m.hall + data << "" + data << m.toilet==null?"":m.toilet + data << m.direction_code==null?"":m.direction_code + data << m.fitment_code==null?"":m.fitment_code + data << "" + data << "" + data << m.p_date==null?"":m.p_date + data << "" + data << "" + data << m.root + data << "" + data << "" + data << "" + data << "" + data << "" + data << "" + data << m.c_date==null?"":m.c_date + data << m.m_date==null?"":m.m_date + data << m.source + data << "" + return data + } + + + def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def retMap = new ConcurrentHashMap() + println city +" 没过滤前:"+ houses.size() + if(houses){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(houses.size()) + + for (int k=0;k60){ + return null + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + return null + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + return null + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + return null + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + return null + } + + if(seawater.avg_price && seawater.avg_price<10){ + return null + } + + } + rowMap.houseId = seawater.id + rowMap.city = city + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(rootMap.get(seawater.root)){ + rowMap.root = rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + + def regionCode = distCodeMap.get(rMap.region) + def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) + def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) + def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) + def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type); + def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); + rowMap.city_code = cityCode + rowMap.region = rMap.region + rowMap.region_code = regionCode + rowMap.district_id = rMap.get("_id").toString() + rowMap.district_name = rMap.name + rowMap.build_year = buildYear; + rowMap.proptype = housePro + rowMap.propyears = propertyYear; + rowMap.estate_type = estateType + rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall); + rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction) + rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment); + rowMap.address = rMap.address + rowMap.build_structure_code = struCode + rowMap.house_structure_code = null//getHouseStructureCode(); + rowMap.source = "FJW" + return rowMap + + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + /* + * 生成一个用不重复的ID + */ + def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) + } + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy new file mode 100644 index 000000000..37c99bc96 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CcbCommonUtils { + + public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] +// public static def citys = ["北京"] + + public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] + + public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") + + public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") + + public static def findCityCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count >1){ + def data = null + for(def db : dbcursor){ + if(db.city_code){ + data = db + break + } + } + return data; + } + } + + public static def findAllDistrictCodeValue(city){ + def q = new BasicDBObject() + q.put("fangjia_city", city) + q.put("district", new BasicDBObject('$ne',"市辖区")) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def map = [:] + for(def db : dbcursor){ + map.put(db.fangjia_district, db.district_code) + } + return map; + } + + public static def findPersonPrice(city, date){ + def q = new BasicDBObject() + q.put("city", city) + q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date))); + def dbcursor = GUIDING_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy new file mode 100644 index 000000000..bafa9e4d8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +/** + * 建行模型估值项目 + * 城市数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class CityBatch extends BaseBatch{ + + static main(args) { + runBatch(new CityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + def index = 1 + for(def city : citys){ + def data = CcbCommonUtils.findCityCodeValue(city) + def map = [:] + map.name = city + if(data){ + map.city_code = data.city_code + map.prov_code = data.province_code + } + def cityObj = findCityInfo(city) + if(cityObj){ + map.city_spell = cityObj.spell + map.city_level = cityObj.city_grade + map.seq = index + map.ifcapital = isCapital(cityObj.province, city) + map.ifmunicipal = ifMunicipal(city) + } + index++ + rows << map + } + + def allDatas = createDatas(rows) + def headers = ["城市ID","城市名称","城市名称拼音","省份ID","国标码","城市级别","显示顺序","是否省会","是否直辖市","备注","创建时间","更新时间","来源公司"] + + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8"); + } + + def createDatas(rows){ + def allRows = [] + rows.each { m -> + def data = [] + data << m.city_code + data << m.name + data << m.city_spell + data << m.prov_code + data << m.city_code + data << m.city_level + data << m.seq + data << CcbCodeUtils.parseFlag(m.ifcapital) + data << CcbCodeUtils.parseFlag(m.ifmunicipal) + data << "" + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "fjw" + allRows << data + } + return allRows; + } + + def findCityInfo(city){ + def q = new BasicDBObject() + q.put("display", city) + return CcbCommonUtils.SERVER_AREA_COLL.findOne(q) + } + + def isCapital(def prov, def city){ + def provCity = CcbCommonUtils.provCityMap.get(prov) + if(provCity && provCity==city){ + return "是" + } + return "否" + } + + def ifMunicipal(def city){ + if(["上海", "北京", "天津", "重庆"].contains(city)){ + return "是" + } + return "否" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy new file mode 100644 index 000000000..be208aebb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql + +/** + * 建行模型估值项目 + * 估价系数数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class EvaluateRateBatch extends BaseBatch{ + + static final def sqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver") + + static main(args) { + runBatch(new EvaluateRateBatch(), args); + } + + @Override + public void process(Object args) { + + def allDatas = createDatas(findEvaluationRate()) + def headers = ["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + def rets = [] + rets << headers + rets.addAll(allDatas) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false); + } + + def createDatas(def datas){ + def allRows = [] + datas.each { m -> + def data = [] + data << m.id.toString() + data << m.rate_type + data << m.rate + data << m.rank + data << "全国" + data << m.rank_remark + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + allRows << data + } + return allRows + } + + private def findEvaluationRate(){ + def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'"; + def rets = [] + sqlInstance.eachRow(sql,{ + rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated] + }); + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy new file mode 100644 index 000000000..234d172b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 建行模型估值项目 + * 人工指导价同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class GuiDingPriceBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new GuiDingPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + + } + + def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false); + + def rets = [] + rets << headers + rets.addAll(rows) + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") + + } + + def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){ + def rets = [] + if(prices){ + for(def ps : prices){ + def dist = distMap.get(ps.region+"##"+ps.name) + if(dist){ + def ret = [] + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + if(dist.region){ + ret << distCodeMap.get(dist.region) + }else{ + ret << "" + } + ret << dist.region + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << ps.guiding_value + ret << ps.guiding_person + ret << "" + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + } + return rets; + } + + def getDistrictMap(def city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + } + return distMap; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy new file mode 100644 index 000000000..4ecac813a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +/** + * 建行模型估值项目 + * 分期数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class StagingDistrictBatch extends BaseBatch{ + + static main(args) { + runBatch(new StagingDistrictBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"]) + def fenQiMap = getFenQiMap(dists); + def rets = createCityData(dists, fenQiMap) + rows.addAll(rets) + println city +" 分期量:"+rets.size() + } + + + def results =[] + def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + results << headers + results.addAll(rows) + + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据") + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8"); + + } + + def createCityData(def dists, def fenQiMap){ + def rets = [] + for(def d : dists){ + if(d.master_district_name && d.master_district_name!="master"){ + def masterDist = fenQiMap.get(d.city+"##"+d.region+"##"+d.master_district_name) + if(masterDist){ + def ret = [] + ret << new ObjectId().toString() + ret << d._id.toString() + ret << masterDist._id.toString() + ret << "" + ret << DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(d.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } +// else{ +// def udata = new BasicDBObject() +// udata.put("master_district_name", null) +// udata.put("m_date", new Date()) +// println d.city +" "+d.name+" "+d._id.toString() +// DbUtils.seaweed.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', udata)) +// } + } + } + return rets; + } + + def getFenQiMap(def dists){ + def fenQiMap = [:] + for(def d : dists){ + if(d.master_district_name){ + fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d) + } + } + return fenQiMap; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy new file mode 100644 index 000000000..d4e252dd5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -0,0 +1,177 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 建行模型估值项目 + * 第三方均价数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class ThirdPartyPriceBatch extends BaseBatch { + + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + + static main(args) { + runBatch(new ThirdPartyPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + def index = 0 + for(def city : citys){ + index++ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"]) + def thirdPriceMap = findThirdPartyPrice(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) + rows.addAll(rets) + println index+" "+city+" "+rets.size() + } + def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false); + + } + + + def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def rets = [] + for(def d : dists){ + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def ret = [] + ret << third._id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + if(d.region){ + ret << distCodeMap.get(d.region) + }else{ + ret << "" + } + ret << d.region + ret << third.s_date + + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + + } + return rets; + } + + + def findThirdPartyPrice(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", curDate) + def dbcursor = AVERAGE_STATISTICS.find(q) + def map = [:] + for(def db : dbcursor){ + map.put(db.region+"##"+db.name, db) + } + return map + } + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy new file mode 100644 index 000000000..af92f0489 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/BestDistrictPriceExport.groovy @@ -0,0 +1,174 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +class BestDistrictPriceExport { + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new BestDistrictPriceExport(); + task.run() + } + + def run(){ + def city = "佛山" + def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + def rets = [] + rets << ["城市","区域","楼盘名称","物业类型","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711","链家价格","可信度"] + def distMap = findCreditHightDistrict2(city, "20210711"); + distMap.each{region, dists-> + + if(region=="三水"){ + def addIndex = 0 + for(def dist : dists){ + if(addIndex <=30){ + def otherPrice = findNearThirdPartyPrice(dist.city, dist.region, dist.district_name) + def stat = findDistrictPrice(dist.city, dist.region, dist.district_name, "20210711") + if(otherPrice){ + def precent = (otherPrice.lianjia_esf_price-stat.avg_price)*100/stat.avg_price + if(Math.abs(precent)>=0 && Math.abs(precent)<=10){ + def ret = [] + ret << "佛山" + ret << dist.region + ret << dist.district_name + ret << dist.estate_type + for(def weekend : months){ + def obj = findDistrictPrice(dist.city, dist.region, dist.district_name, weekend) + if(obj){ + ret << obj.avg_price + }else{ + ret << "" + } + } + ret << otherPrice.lianjia_esf_price + ret << stat.credit + rets << ret + addIndex++ + } + } + } + } + } + } + ExcelUtils.write("d:\\数据\\福州\\佛山楼盘均价数据-三水.xlsx",rets, "价格") + } + + + def findDistrictPrice(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def obj = DIST_SELL_STAT.findOne(q) + return obj + } + + def findCreditHightDistrict(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def typeMap = retMaps.get(d.region) + if(!typeMap){ + typeMap = [:] + retMaps.put(d.region, typeMap) + } + + def rets = typeMap.get(d.estate_type) + if(!rets){ + rets = [] + typeMap.put(d.estate_type, rets) + } + + def count = 0 + typeMap.each {k,v-> + if(v){ + count = count + v.size() + } + } + + if(d.estate_type=="别墅" && rets.size()<=5 && count<=20){ + def m = [:] + m.city = d.city + m.region = d.region + m.district_name = d.district_name + rets << m + }else if(rets.size() <= 20){ + def m = [:] + m.city = d.city + m.region = d.region + m.district_name = d.district_name + rets << m + } + + def total = 0 + retMaps.each {k,m-> + m.each {t,v-> + total = total + v.size() + } + } + if(total>100){ + break; + } + } + + def rows = [] + retMaps.each {k,m-> + m.each {t,v-> + rows.addAll(v) + } + } + return rows + } + + + def findCreditHightDistrict2(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyyMMdd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + def m = [:] + m.city = d.city + m.region = d.region + m.estate_type = d.estate_type + m.district_name = d.district_name + rets << m + } +// +// def rows = [] +// retMaps.each {k,v-> +// rows.addAll(v) +// } + return retMaps; + } + + def findNearThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", 202106) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + return AVERAGE_STATISTICS.findOne(q) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy new file mode 100644 index 000000000..4687872d3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SourceDistDataMatch.groovy @@ -0,0 +1,265 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.apache.lucene.queryparser.classic.QueryParser + +class SourceDistDataMatch { + + static main(args) { + def task = new SourceDistDataMatch(); +// task.run() + task.mapRun() +// task.filterName() +// task.filterType() + } + + def run(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(obj){ + d['R'] = 1 + }else{ + def distCount = findHouseCount(d['A'], d['B'], [d['C']], 20190709, 20210709) + if(distCount){ + d['W'] = distCount + } + } + },0) + } + + def mapRun(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N'] && !d['F']){ + def dist = ["city":d['A'], "region":d['B'], "name":d['C'], "address":d['D']] + def locationMap = matchMap(dist) + if(locationMap){ + d['F'] = locationMap.source + d['G'] = locationMap.region + d['H'] = locationMap.mapName + d['I'] = locationMap.address + d['J'] = locationMap.type + d['K'] = locationMap.lng + d['L'] = locationMap.lat + d['M'] = locationMap.score + } + } + },0) + } + + def filterName(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N']) { + def name = d['C'] + if (name) { + def flag = NameProcess.filterName(name) + if (!flag) { + d['N'] = 4 + } + } + } + },0) + + } + + def filterType(){ + def path = "D:\\数据\\小区\\上海第三方均价.xlsx" + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['N']){ + def type = d['J'] + if(type && (type.contains("房地产") || (type.contains("商务住宅") && type.contains("商住两用")) || (type.contains("住宅区") && !type.contains("科教") && !type.contains("生活") && !type.contains("体育") && !type.contains("政府") && !type.contains("公司") && !type.contains("购物") && !type.contains("住宿")))){ + + }else if(type && type.contains("商务住宅") && !type.contains("商住两用")){ + def count = 0 + if(d['S'] && d['S']!='null'){ + count = d['S'].toDouble().toInteger() + } + if(count<10){ + d['N'] = 3 + } + }else{ + if(type){ + d['N'] = 3 + } + } + } + },0) + } + + + def matchMap(def dist){ + def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") + if(!gaodeRest && dist.address){ + gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德") + } + def gdlocationMap = null + def bdlocationMap = null + if(gaodeRest){ + gdlocationMap = getMapLocation2(dist, gaodeRest); + if(gdlocationMap){ + gdlocationMap.source = "高德" + } + } + + def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + if(baiduRest){ + bdlocationMap = getMapLocation2(dist, baiduRest); + if(bdlocationMap){ + bdlocationMap.source = "百度" + } + } + + if(gdlocationMap && bdlocationMap){ + if(gdlocationMap.score>=bdlocationMap.score){ + return gdlocationMap + }else{ + return bdlocationMap + } + }else if(gdlocationMap){ + return gdlocationMap + }else if(bdlocationMap){ + return bdlocationMap + } + return null + } + + + + def findHouseCount(def city, def region, def alias, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } + + def getMapLocation2(def dist, def rets){ + def data1 = findSimilarData(dist, rets, true) + def data2 = findSimilarData(dist, rets, false) + println data1 + println data2 + if(data1 && data1.score>1){ + return data1 + }else if(data2 && data2.score>1){ + return data2 + } + return null + } + + def findSimilarData(def dist, def rets, def regionImpr){ + def lat = null + def lng = null + def region = null + def mapName = null + def address = null + def score = 0 + def type = null + def maxScore = 0 + def standardName = null + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + if(["公交车站"].contains(r.type)){ + continue + } + if(regionImpr && !r.district){ + continue + } + if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){ + continue + } + + def name = nameReplace(r.name, dist.city) + + def distName = nameReplace(dist.name, dist.city) + + def score3 = 0.0 + if(r.district && dist.region && r.district.contains(dist.region)){ + score3 = 0.2 + } + + def score1 = 0.0; + def score2 = 0.0 + if(distName && (distName.contains("号") || distName.contains("弄")) && (distName.contains("路") || distName.contains("街") || distName.contains("大道"))){ + if(r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(distName, r.address)/2 + } + }else{ + score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2; + if(dist.address && r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2 + } + } + + score3 = score1 + score2 + if(r.type && r.type.contains("住宅")){ + score3 = score3 + 0.2 + } + if(score3 >maxScore){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + standardName = name + address = r.address + type = r.type + score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10) + maxScore = score3 + } + println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3 + + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName] + } + return null + } + + def nameReplace(def s, city){ + if(s.contains("(") && s.contains(")")){ + def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")")) + if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){ + s = s.toString().substring(0, s.indexOf("(")) + } + } + def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim() + if(name.startsWith(city)){ + def newName = name.replace(city+"市","").replace(city,"") + if(newName && newName=="花园"){ + return name + }else{ + name = newName + } + } + return name + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy new file mode 100644 index 000000000..40748a9fd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.gj.region + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class MapAreaUpdate { + static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + static main(args) { + def task = new MapAreaUpdate() + task.run() + } + + def run(){ + def rets = findAllCity() + for(def city : rets){ + def retMap = findCityAndProvCode(city) + if(retMap){ + update(retMap) + } + } + } + + def update(retMap){ + def q = new BasicDBObject() + q.put("fangjia_city",retMap.fangjia_city) + q.put("district", "市辖区") + def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q) + if(obj){ + def udata = new BasicDBObject() + udata.put("city_code", retMap.city_code) + udata.put("province_code", retMap.province_code) + FANGJIA_OFFICIAL_AREA_MAP.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata), true, false) + } + } + + def findCityAndProvCode(def city){ + def q = new BasicDBObject() + q.put("fangjia_city",city) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def map = [:] + for(def d : dbcursor){ + if(d.city_code && d.province_code && d.district_code){ + map.put("fangjia_city",d.fangjia_city) + map.put("city_code",d.city_code) + map.put("province_code",d.province_code) + break + } + } + return map; + } + + def findAllCity(){ + def q = new BasicDBObject() + q.put("fangjia_district",null) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def citys = [] + for(def d : dbcursor){ + citys << d.fangjia_city + } + citys.unique() + return citys + } +}