diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index e075fbd32..582a71f23 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -258,8 +258,9 @@ public class BaiDuApiUtils { params.put("coords", lng.toString() + "," + lat.toString()); params.put("from", String.valueOf(from)); HttpRequest request = new HttpRequest(); + String result = null; try{ - String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); + result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); JSONObject json = JSONObject.fromObject(result); while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){ changeAK(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy new file mode 100644 index 000000000..93864962e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -0,0 +1,88 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 建行模型估值项目 + * 小区建筑类型数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataCommbuildtypeBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + runBatch(new BigdataCommbuildtypeBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rets = [] + for(def city : citys){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"]) + def rows = createDatas(cityCodeDatas, distCodeMap, dists) + rets.addAll(rows) + println city +" "+rows.size() + } + def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false); + println "txt 生成" + + } + + def createDatas(cityCodeDatas, distCodeMap, dists){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i> FITMENTS_TYPES_MAPS = new HashMap>() - static{ - FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) - FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) - FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) - FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) - } - - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) - - def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] - - static main(args) { - runBatch(new BigdataForsaleBatch2(), args); - } - - @Override - public void process(Object args) { - def citys = CcbCommonUtils.citys; - for(def city : citys){ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def distMap = getDistrictMap(city) - exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731); - } - } - - def getDistrictMap(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); - def distMap = [:] - for(def d : districts){ - distMap.put(d.region+"##"+d.name, d) - if(d.alias){ - d.alias.each{name-> - if(!distMap.containsKey(d.region+"##"+name)){ - distMap.put(d.region+"##"+name, d) - } - } - } - } - return distMap; - } - - def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){ - println city +" 小区key:"+distMap.size() - def cityCode = cityCodeDatas.city_code - def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap) - def allDatas = createSellDatas(city, retMap); - def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true); - println "txt文件生成 " - } - - - - def createSellDatas(def city, def map){ - - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(map.size()) - println city +"总房源量:"+map.size() - - def datas = [] - map.each{k,m-> - fixedThreadPool.execute(new Runnable() { - void run() { - try { - totalCount.addAndGet(1) - datas << converData(m) - println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString() - } catch (Exception e) { - e.printStackTrace() - } finally { - latch.countDown() - } - }; - }) - } - latch.await() - return datas ; - } - - def converData(def m){ - def data = [] - data << m.city_code - data << m.houseId - data << "" - data << m.city - data << m.region_code - data << m.region - data << m.district_id - data << m.district_name - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << m.address==null?"":m.address==null - data << m.build_year==null?"":m.build_year - data << m.proptype==null?"":m.proptype - data << m.propyears==null?"":m.propyears - data << m.avg_price==null?"":m.avg_price - data << m.total_price==null?"":m.total_price - data << "" - data << m.floor==null?"":m.floor - data << m.height==null?"":m.height - data << m.estate_type==null?"":m.estate_type - data << m.house_structure_code==null?"":m.house_structure_code - data << m.build_structure_code==null?"":m.build_structure_code - data << m.area==null?"":m.area - data << "" - data << m.house_type==null?"":m.house_type - data << m.room==null?"":m.room - data << m.hall==null?"":m.hall - data << "" - data << m.toilet==null?"":m.toilet - data << m.direction_code==null?"":m.direction_code - data << m.fitment_code==null?"":m.fitment_code - data << "" - data << "" - data << m.p_date==null?"":m.p_date - data << "" - data << "" - data << m.root - data << "" - data << "" - data << "" - data << "" - data << "" - data << "" - data << m.c_date==null?"":m.c_date - data << m.m_date==null?"":m.m_date - data << m.source - data << "" - return data - } - - - def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){ - - QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); - QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); - - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) - .must(date_range).must(area_range) - - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); - def retMap = new ConcurrentHashMap() - println city +" 没过滤前:"+ houses.size() - if(houses){ - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(houses.size()) - - for (int k=0;k60){ - return null - } - -// if(!seawater.room){ -// continue; -// } - if(!seawater.room){ - seawater.room = null - } - - if(seawater.room){ - if(seawater.room>=5 || seawater.room<1){ - return null - } - } - - if(!seawater.hall){ - seawater.hall = null - } - if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ - return null - } - if(!seawater.toilet){ - seawater.toilet = null - } - if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ - return null - } - - - def floor = parseFloor(seawater.floor_info) - if(seawater.height && floor && floor.isNumber()){ - if(floor.toInteger()>seawater.height){ - floor = null - seawater.height = null - } - - } - - if(!floor){ - seawater.floor = null - } - if(floor && floor.toInteger()<1){ - floor = null - } - - def rowMap = [:]; - if(cat == "rent"){ - rowMap.rent_type = seawater.rent_type - if(seawater.avg_price && seawater.avg_price>1000){ - return null - } - - if(seawater.avg_price && seawater.avg_price<10){ - return null - } - - } - rowMap.houseId = seawater.id - rowMap.city = city - //rowMap.region =seawater.region - //rowMap.address = seawater.address - - - rowMap.area = seawater.area - rowMap.total_price = seawater.total_price - rowMap.avg_price = seawater.avg_price - rowMap.height = seawater.height=="null"?null:seawater.height - if(rowMap.height && rowMap.height<1){ - rowMap.height = null - } - rowMap.floor = floor=="null"?null:floor - rowMap.room = seawater.room=="null"?null:seawater.room - rowMap.hall = seawater.hall=="null"?null:seawater.hall - rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet - if(rootMap.get(seawater.root)){ - rowMap.root = rootMap.get(seawater.root) - }else{ - rowMap.root = "99" - } - - def direction = parseDirection(seawater.direction) - rowMap.direction = direction=="null"?null:direction - - def fitment = parseFitment(seawater.fitment) - rowMap.fitment = fitment=="null"?null:fitment - - - - if(seawater.p_date instanceof String){ - seawater.p_date = seawater.p_date.toInteger() - } - if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ - rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") - rowMap.c_date = rowMap.p_date - rowMap.m_date = rowMap.p_date - }else{ - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) - } - - def regionCode = distCodeMap.get(rMap.region) - def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) - def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) - def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) - def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type); - def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); - rowMap.city_code = cityCode - rowMap.region = rMap.region - rowMap.region_code = regionCode - rowMap.district_id = rMap.get("_id").toString() - rowMap.district_name = rMap.name - rowMap.build_year = buildYear; - rowMap.proptype = housePro - rowMap.propyears = propertyYear; - rowMap.estate_type = estateType - rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall); - rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction) - rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment); - rowMap.address = rMap.address - rowMap.build_structure_code = struCode - rowMap.house_structure_code = null//getHouseStructureCode(); - rowMap.source = "FJW" - return rowMap - - } - - def getPublicDate(def pDate){ - - if(pDate>=20181001 && pDate<20200201){ - return DateUtils.getRandomDate("2018-10-01", "2020-02-01") - } - def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") - return pdate; - } - - static def parseFloor(def floorDesc){ - if(floorDesc){ - floorDesc = floorDesc.replaceAll("[\\s ]+", "") - if(floorDesc){ - Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); - Matcher matcher1 = pat1.matcher(floorDesc); - if (matcher1.find()) { - return matcher1.group(1) - } - pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); - matcher1 = pat1.matcher(floorDesc); - if (matcher1.find()) { - return matcher1.group(1) - } -// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ -// return "中层" -// } -// -// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ -// return "高层" -// } -// -// -// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ -// return "低层" -// } - if(floorDesc.isNumber()){ - return floorDesc; - } - } - } - return null; - } - - private String csvHandlerStr(String str) { - //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 - String tempDescription=str; - //如果有逗号 - if(str.contains(",")||str.contains(",")){ - //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 - if(str.contains("\"")){ -// tempDescription=str.replace("\"", "\"\""); - tempDescription=str.replace("\"", ""); - } - //在将逗号转义 - tempDescription="\""+tempDescription+"\""; - } - return tempDescription; - } - - - static String parseFitment(String fitmentDesc) { - if (fitmentDesc == null) { - return null - } - fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") - if (fitmentDesc.equals("非毛坯")) { - return null - } - Iterator>> it = FITMENTS_TYPES_MAPS - .entrySet().iterator() - Map.Entry> entry - while (it.hasNext()) { - entry = it.next() - if (fitmentDesc.contains(entry.getKey())) { - return entry.getKey() - } - for (String fitmentKey : entry.getValue()) { - if (fitmentDesc.contains(fitmentKey)) { - return entry.getKey() - } - } - } - return null - - } - - static String parseDirection(String directionDesc) { - if (directionDesc == null) { - return null - } - directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") - StringBuffer sb = new StringBuffer() - for (int i = 0; i < directionDesc.length(); i++) { - String c = directionDesc.substring(i, i + 1) - if (DIRECTIONS.contains(c)) { - if (directionDesc.substring(i).startsWith(c + "#_#")) { - sb.append(c + " ") - } else { - sb.append(c) - } - } - } - for (String d : DIRECTIONS) { - if (sb.toString().startsWith(d)) { - return d - } - } - for (String d : DIRECTIONS) { - if (sb.toString().contains(d)) { - return d - } - } - return null - - } - - /* - * 生成一个用不重复的ID - */ - def GenNonDuplicateID(randomLength){ - return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) - } - - - - def marge(){ - File file = new File("D:\\数据\\建行\\新建文本文档.txt"); - def ids = new HashSet(); - def index = 0 - - file.eachLine{it-> - index++ - if(index>1){ - def arrs = it.split("\\|@\\|") - def id = arrs[1] - if(!ids.contains(id)){ - ids << id - } - } - } - println index - println ids.size() - } - -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 37c99bc96..c11a06cfa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -7,8 +7,8 @@ import com.mongodb.DBCollection class CcbCommonUtils { - public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] -// public static def citys = ["北京"] + public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"] +// public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -18,6 +18,11 @@ class CcbCommonUtils { public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static def findCityCodeValue(city){ def q = new BasicDBObject() q.put("fangjia_city", city) @@ -58,4 +63,46 @@ class CcbCommonUtils { } return rets } + + public static def findTradesDetail(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + def lists = [] + for(def d : dbcursor){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } + } + return lists + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy index bafa9e4d8..ef52b58c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -52,7 +52,7 @@ class CityBatch extends BaseBatch{ rets << headers rets.addAll(allDatas) ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8"); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy new file mode 100644 index 000000000..399e3c53a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -0,0 +1,21 @@ +package com.fangjia.script.ccbevaluation +import groovy.sql.Sql + +def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +def citys = CcbCommonUtils.citys; +def rows = [] + +for(def city : citys) { + def data = CcbCommonUtils.findCityCodeValue(city) + def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; + + def count = 0 + sqlInstance.execute(sql,{ + count = it.count + }); + println city +" "+count +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy index aece3ef15..d61bfb2cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy @@ -74,20 +74,20 @@ class AddThirdPriceDistrict { static main(args) { def task = new AddThirdPriceDistrict(); -// task.run() + task.run() - def mapName = "锦华星光苑3,4幢" +// def mapName = "锦华星光苑3,4幢" //// println task.pattern("港新花苑A1区"); - println mapName.replaceAll("\\w+-?\\w+(幢)", "") - println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") +// println mapName.replaceAll("\\w+-?\\w+(幢)", "") +// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "") } def run(){ -// exportDatas(); + exportDatas(); // getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); // addInfo(); // addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx") // addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx") - addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); +// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); } def addCrawlMapInfo( def path){ @@ -135,29 +135,32 @@ class AddThirdPriceDistrict { def exportDatas(){ - def city = "无锡" - def datas = findMatchErrorDatas(city); - println "共【"+datas.size()+"】条" - datas = filterName(datas); - println "过滤后共【"+datas.size()+"】条" - def resMap = groupByName(datas); - def rets = [] - rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"] - resMap.each {k,v-> - def ret = [] - def arrs = k.split("##") - ret << arrs[0] - ret << arrs[1] - if(arrs.size()==3 && arrs[2]){ - ret << arrs[2] - }else{ - ret << "" + //跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞", + def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + for(def city : citys){ + def datas = findMatchErrorDatas(city); + println "共【"+datas.size()+"】条" + datas = filterName(datas); + println "过滤后共【"+datas.size()+"】条" + def resMap = groupByName(datas); + def rets = [] + rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"] + resMap.each {k,v-> + def ret = [] + def arrs = k.split("##") + ret << arrs[0] + ret << arrs[1] + if(arrs.size()==3 && arrs[2]){ + ret << arrs[2] + }else{ + ret << "" + } + ret << resAddrMap.get(k) + ret << v.join(",") + rets << ret } - ret << resAddrMap.get(k) - ret << v.join(",") - rets << ret + ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后") } - ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后") } def filterName(datas){ @@ -465,7 +468,7 @@ class AddThirdPriceDistrict { def findMatchErrorDatas(def city){ def q = new BasicDBObject() q.put("city",city) - q.put("s_date",202005) + q.put("s_date",new BasicDBObject('$gte', 202101)) // q.put("name","博济聚创科技园") // q.put("source","贝壳找房") def dbcursor = matchErrorLogColl.find(q) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 00c1ec513..a8b29aff0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -56,11 +56,11 @@ class GaoDeMapCrawl extends BaseBatch{ // def prefix = "road_" // def categorys = ["190301"]//道路相关 -// def prefix = "village_" -// def categorys = ["190108","190109"]//村庄 + def prefix = "village_" + def categorys = ["190108","190109"]//村庄 - def prefix = "company_" - def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] +// def prefix = "company_" +// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] def totalCount = new AtomicLong() static main(args) { @@ -118,7 +118,8 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" - def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys =["无锡"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy new file mode 100644 index 000000000..747ddf078 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ExportDistrictInfo.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + + +def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"] + +def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"] +def rets = [] +rets << headers + +for(def s : districts){ + def arrs = s.split("_") + def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2]) + def ret = [] + ret << s + if(m){ + ret << m.city + ret << m.district + ret << m.name + if(m.alias){ + ret << m.alias.toString().replace("[","").replace("]","") + }else{ + ret << "" + } + ret << m.address + if(m.addressAlias){ + ret << m.addressAlias.toString().replace("[","").replace("]","") + }else{ + ret << "" + } + + ret << m.avgPrice + ret << m.lng + ret << m.lat + ret << m.block + ret << m.loopLine + ret << m.propertyType + ret << m.buildingType + ret << m.commercialArea + ret << m.totalUnitCount + ret << m.managementFee + ret << m.parking + ret << m.parkingDescription + ret << m.ownershipPeriod + ret << m.ownershipType + ret << m.startBuildTime + ret << m.startSellingTime + ret << m.finishBuildTime + ret << m.moveinTime + ret << m.floorSpace + ret << m.constructionSpace + ret << m.plotRatio + ret << m.greenPercentage + ret << m.preSalePermit + ret << m.developer + ret << m.propertyManagement + ret << m.decorationStandard + ret << m.buildingStructure + ret << m.supportingFacilities + ret << m.transportation + ret << m.relatedInfo + ret << m.description + ret << m.salesOfficeAdd + ret << m.salesPhoneNumber + ret << m.discount + ret << m.buildingCount + ret << m.finishYear + + } + rets << ret +} +ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据") + + + +