From 3764b2876ba043767c47fbb99922b89d0d6eae1c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 14 Mar 2021 00:08:52 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E6=95=B0=E6=8D=AE=E5=AF=BC?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-core/pom.xml | 4 +- .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 14 + .../fangjia/script/gj/ccb/HouseExport.groovy | 705 ++++++++++++++++++ .../com/fangjia/script/util/DateUtils.groovy | 13 + .../script/util/FangJiaAreaMapUtils.groovy | 4 + 5 files changed, 738 insertions(+), 2 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy diff --git a/projects/fangjia-core/pom.xml b/projects/fangjia-core/pom.xml index 7f01b5fce..2a0d00ddd 100644 --- a/projects/fangjia-core/pom.xml +++ b/projects/fangjia-core/pom.xml @@ -84,13 +84,13 @@ org.sothis sothis-core - 0.2.0-SNAPSHOT + 0.2.0 provided org.sothis sothis-web - 0.2.0-SNAPSHOT + 0.2.0 provided diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 72117edad..e3cb5bfee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -147,4 +147,18 @@ class CcbCodeUtils { static def getHouseStructureCode(){ return null } + + //租赁方式 整租:0 合租:1 转租:2 + static def getRentTypeCode(def rentType){ + if(rentType){ + if(rentType.contains("整租")){ + return 0; + }else if(rentType.contains("合租")){ + return 1; + }else if(rentType.contains("转租")){ + return 2; + } + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy new file mode 100644 index 000000000..a3bad5266 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -0,0 +1,705 @@ +package com.fangjia.script.gj.ccb + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.commons.lang.StringUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource + +import java.sql.ResultSet +import java.text.SimpleDateFormat +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern +import com.fangjia.script.util.FangJiaAreaMapUtils + +class HouseExport { + + + public static JdbcDataSource dataSource = new JdbcDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + static def totalCount = new AtomicLong() + + def cityCodeMap = ["北京":"110100","重庆":"500100", "哈尔滨":"230100"] + def regionCodeMap = [:] + + static main(args) { + def task = new HouseExport(); + + task.run() + + } + + def run(){ + + def city = "重庆" + getRegionCodeByCity(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); + + + exportSellCSV(city, districts); +// exportRentCSV(city, districts); + + } + + def exportSellCSV(city, districts){ + final CountDownLatch latch = new CountDownLatch(districts.size()) + println city +" 小区量:"+districts.size() + def datas = [] + + def retMap = new ConcurrentHashMap() + for(def i=0;i0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } + + + + + + + def writeCSV(def list, def path) { + File file = new File(path); + if (file.exists()) { + file.delete(); + } + file.append("城市ID|@|", "UTF-8") + file.append("出售ID|@|", "UTF-8") + file.append("挂牌房屋ID|@|", "UTF-8") + file.append("城市名称|@|", "UTF-8") + file.append("行政区ID|@|", "UTF-8") + file.append("行政区名称|@|", "UTF-8") + file.append("小区ID|@|", "UTF-8") + file.append("小区名称|@|", "UTF-8") + file.append("楼栋ID|@|", "UTF-8") + file.append("楼栋编号|@|", "UTF-8") + file.append("楼栋名称|@|", "UTF-8") + file.append("单元ID|@|", "UTF-8") + file.append("单元编号|@|", "UTF-8") + file.append("单元名称|@|", "UTF-8") + file.append("户ID|@|", "UTF-8") + file.append("详细地址|@|", "UTF-8") + file.append("建筑年代|@|", "UTF-8") + file.append("权属|@|", "UTF-8") + file.append("权属年限|@|", "UTF-8") + file.append("单价|@|", "UTF-8") + file.append("总价|@|", "UTF-8") + file.append("出售付款方式|@|", "UTF-8") + file.append("所在楼层|@|", "UTF-8") + file.append("总楼层|@|", "UTF-8") + file.append("建筑类型|@|", "UTF-8") + file.append("室内结构|@|", "UTF-8") + file.append("建筑结构|@|", "UTF-8") + file.append("建筑面积|@|", "UTF-8") + file.append("套内面积|@|", "UTF-8") + file.append("户型|@|", "UTF-8") + file.append("室|@|", "UTF-8") + file.append("厅|@|", "UTF-8") + file.append("厨|@|", "UTF-8") + file.append("卫|@|", "UTF-8") + file.append("朝向|@|", "UTF-8") + file.append("装修情况|@|", "UTF-8") + file.append("是否有电梯|@|", "UTF-8") + file.append("几梯几户|@|", "UTF-8") + file.append("发布时间|@|", "UTF-8") + file.append("到期时间|@|", "UTF-8") + file.append("是否首发|@|", "UTF-8") + file.append("房源数据来源|@|", "UTF-8") + file.append("中介公司ID|@|", "UTF-8") + file.append("中介公司名称|@|", "UTF-8") + file.append("发布人|@|", "UTF-8") + file.append("发布人电话|@|", "UTF-8") + file.append("描述信息|@|", "UTF-8") + file.append("备注|@|", "UTF-8") + file.append("创建时间|@|", "UTF-8") + file.append("更新时间|@|", "UTF-8") + file.append("来源公司|@|", "UTF-8") + + file.append(System.getProperty("line.separator")); + def index = 0 + for(def m : list) { + index ++ + if(index<1000){ + file.append(m.city_code + "|@|", "UTF-8") + file.append(m.houseId + "|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append(m.city+"|@|", "UTF-8") + file.append(m.region_code+"|@|", "UTF-8") + file.append(m.region+"|@|", "UTF-8") + file.append(m.district_id+"|@|", "UTF-8") + file.append(m.district_name+"|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append(m.address+"|@|", "UTF-8") + file.append(m.build_year+"|@|", "UTF-8") + file.append(m.proptype+"|@|", "UTF-8") + file.append(m.propyears+"|@|", "UTF-8") + file.append(m.avg_price+"|@|", "UTF-8") + file.append(m.total_price+"|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append(m.floor+"|@|", "UTF-8") + file.append(m.height+"|@|", "UTF-8") + file.append(m.estate_type+"|@|", "UTF-8") + file.append( m.house_structure_code+"|@|", "UTF-8") + file.append(m.build_structure_code+"|@|", "UTF-8") + file.append(m.area+"|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append(m.house_type+"|@|", "UTF-8") + file.append(m.room+"|@|", "UTF-8") + file.append(m.hall+"|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append(m.toilet+"|@|", "UTF-8") + file.append(m.direction_code+"|@|", "UTF-8") + file.append(m.fitment_code+"|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append(m.p_date+"|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append("|@|", "UTF-8") + file.append(m.c_date+"|@|", "UTF-8") + file.append(m.m_date+"|@|", "UTF-8") + file.append(m.source+"|@|", "UTF-8") + + file.append(System.getProperty("line.separator")); + } + + } + } + + + def getRegionCodeByCity(def city){ + def datas = FangJiaAreaMapUtils.convertToOfficials(city); + for(def d : datas){ + regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code) + } + } + + + def findHouses(def city, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(seawater.room && (seawater.room>=5 || seawater.room<1)){ + continue; + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + } + rowMap.houseId = seawater.id + rowMap.city = city + rowMap.region =seawater.region + rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + rets << rowMap + } + } + return rets; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ + return "中层" + } + + if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ + return "高层" + } + + + if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ + return "低层" + } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index cbda72dbc..8bc5d9198 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -765,6 +765,19 @@ class DateUtils { long t = currMills + mills; return new Date(t) } + + static String getConverDate(def mills){ + String s = null + if(mills instanceof String){ + s = mills.replace("T"," ").replace(".000Z","") + }else if(mills instanceof Long){ + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date date = new Date(mills); + s = dateFormat.format(date); + } + + return s + } static def setToday(d){ Calendar calendar = Calendar.getInstance(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/FangJiaAreaMapUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/FangJiaAreaMapUtils.groovy index cf4ec626d..945b6c56e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/FangJiaAreaMapUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/FangJiaAreaMapUtils.groovy @@ -19,4 +19,8 @@ class FangJiaAreaMapUtils { return (BasicDBObject)FANGJIA_OFFICIAL_AREA_MAP.findOne(new BasicDBObject("district_code", districtCode.toString())) } + static def convertToOfficials(String city){ + return FANGJIA_OFFICIAL_AREA_MAP.find(new BasicDBObject("fangjia_city", city).append("district_code", new BasicDBObject('$ne', null))) + } + }