diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index dc7821343..4d50b3661 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -7,7 +7,7 @@ import java.sql.ResultSet class EsSearchUtils { public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1f5247429..3fd9abe32 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -394,6 +394,7 @@ class FiledStrand { } return estateCharege; } + static def processHouseholdCount(def household_count, def estateType2){ if(household_count){ household_count = household_count.toInteger() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 4ec26d7db..b5efa4f51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -29,6 +29,8 @@ class OpenApiUtils { rMap.region = dto.district rMap.name = dto.name rMap.id = dto.id + rMap.lng = dto.lng + rMap.lat = dto.lat if(dto.address && dto.address!="null"){ rMap.address = dto.address } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy index f8c8c0d5c..cbf9f5e12 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy @@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils class WuXiAddressTest { static main(args) { def task = new WuXiAddressTest() - task.run() + //task.run() + task.addAddress() + } + + def addAddress(){ + ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d-> + def prov = d['C'] + def city = d['D'] + def region = d['E'] + def address = d['B'] + def road = d['J'] + def street = d['R'] + + def newAddress = prov+"省"+city +"市" + if(region == "惠山"){ + newAddress = newAddress + region +"区" + }else{ + newAddress = newAddress + region +"市" + } + + d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "") + def dist = d['U'] + + def s1 = null + if(road){ + s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","") + newAddress = newAddress + street + dist + road + s1 + }else if(d['L']){ + s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","") + newAddress = newAddress + street + dist + d['L'] + s1 + } + + d['V'] = newAddress + },0) } def run(){ @@ -18,6 +51,8 @@ class WuXiAddressTest { // // } + + ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d-> if(!d['S']){ def locationStr = getLocationInfo(d['D'], d['Q']) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 0c2be14c8..d8ccfa02e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,11 +47,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-05-01" + static def statDate = "2021-06-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 89dff40d4..7a766b668 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -7,6 +7,13 @@ import java.util.regex.Pattern class CcbCodeUtils { + //否:0 是:1 + static def parseFlag(String flag){ + if(flag && flag == "是"){ + return 1; + } + return 0; + } static def parseBuildYear(String buildDate) { if(buildDate){ @@ -53,6 +60,44 @@ class CcbCodeUtils { return null } +// 高层塔楼 0101 增加“超高层” 0108 +// 高层板楼 0102 增加“老洋房” 0109 +// 小高层板楼 0103 增加“花园洋房” 0110 +// 多层板楼 0104 增加“四合院” 0111 +// 独栋别墅 0105 增加“老式别墅洋房” 0112 +// 联排别墅 0106 +// 叠拼别墅 0107 +// 其他建筑形式 0199 + + static def parseBuildType(def buildType){ + if(buildType){ + if(buildType.contains("老式别墅洋房")){ + return "0112" + }else if(buildType.contains("四合院")){ + return "0111" + }else if(buildType.contains("花园洋房")){ + return "0110" + }else if(buildType.contains("老洋房")){ + return "0109" + }else if(buildType.contains("叠拼")){ + return "0107" + }else if(buildType.contains("联排")){ + return "0106" + }else if(buildType.contains("独栋")){ + return "0105" + }else if(buildType.contains("多层")){ + return "0104" + }else if(buildType.contains("超高层")){ + return "0108" + }else if(buildType.contains("小高层")){ + return "0103" + }else if(buildType.contains("高层")){ + return "0102" + } + } + return "0199" + } + //朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399 static def getDirectionCode(def direction){ @@ -161,4 +206,19 @@ class CcbCodeUtils { } return 0; } + + + //房屋用途 + static def parseHouseUse(String houseUse){ + if(houseUse){ + if(houseUse.contains("住宅")){ + return 11; + }else if(houseUse.contains("商住")){ + return 13; + }else{ + return 99 + } + } + return 99 + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 09f80b9da..9382cb398 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -3,13 +3,13 @@ package com.fangjia.script.gj.ccb import org.apache.commons.lang.StringUtils class FileUtils { - static def batchCreateFile(ret, fileName, headers,charset){ + static def batchCreateFile(ret, fileName, headers,charset, isAppend){ File file = new File(fileName); - if (file.exists()) { - file.delete(); - } +// if (file.exists()) { +// file.delete(); +// } - OutputStream out = new FileOutputStream(fileName); + OutputStream out = new FileOutputStream(fileName, isAppend); def sb = new StringBuffer(); sb.append(headers.join("|@|")) @@ -20,18 +20,18 @@ class FileUtils { def index = 0 for(def row : ret){ - index++ - sb.append(StringUtils.join(row.toArray(), "|@|")) - sb.append(System.getProperty("line.separator")); - println index + if(row){ + index++ + sb.append(StringUtils.join(row.toArray(), "|@|")) + sb.append(System.getProperty("line.separator")); - if(index%10000==0){ - println index+"开始写入CSV "+new Date().toLocaleString() - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - println index+"结束写入CSV "+new Date().toLocaleString() + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } } - } if(sb.length()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy index d059de4ae..a56e2079d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.data import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -10,22 +11,23 @@ import com.mongodb.DBCollection class RegionPriceExport { DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com") + DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") static main(args) { def task = new RegionPriceExport(); task.run() } def run(){ - def months = [202010,202011,202012,202101,202102,202103] - def regions = CommonUtils.getAllRegions("福州","region") + def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + def regions = CommonUtils.getAllRegions("佛山","region") def rets = [] - rets << ["城市","区域","202010","202011","202012","202101","202102","202103"] + rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] regions.each{region,values-> def data = [] - data << "福州" + data << "佛山" data << region for(def month :months){ - def dbo = findPrice("福州", region, month); + def dbo = findWeekendPrice("佛山", region, month); if(dbo){ data << dbo.avg_price }else{ @@ -34,7 +36,7 @@ class RegionPriceExport { } rets << data } - ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格") + ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格") } def findPrice(city, region, month){ @@ -45,4 +47,14 @@ class RegionPriceExport { def dbo = REGION_SELL_STAT.findOne(q) return dbo } + + def findWeekendPrice(city, region, month){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("cat", "区域") + q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) + def dbo = REGION_STAT.findOne(q) + return dbo + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index 6a44d4139..6f9219576 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.gj.district import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject @@ -13,7 +15,7 @@ class ExportData { def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol(); public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -24,8 +26,8 @@ class ExportData { static main(args) { def task = new ExportData(); - task.exportThird() -// task.exportMap(); +// task.exportThird() + task.exportMap(); } def exportMap(){ @@ -51,19 +53,18 @@ class ExportData { //"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特" //宁夏 ----- 中卫 固原 石嘴山 吴忠 银川 //青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁 - def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州", - "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮", - "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"] - //"沈阳","大连","抚顺","鞍山" - //"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆" - //"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛", - //"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州", - //"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城", +// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州", +// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮", +// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"] + def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]; + + for(def city : citys){ + def path = "D:\\数据\\小区\\地图\\${city}.xlsx"; + def datas = findAllDatas(city, null) + println "${city}共计:"+datas.size() + ExcelUtils.write(path, datas, "数据") + } - def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx"; - def datas = findAllDatas(citys, null) - println "共计:"+datas.size() - ExcelUtils.write(path, datas, "数据") } @@ -98,9 +99,9 @@ class ExportData { - def findAllDatas(def citys, def pname){ + def findAllDatas(def city, def pname){ final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1") + sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'") // sqlBuffer.append(" and pname = '"+pname+"'") // sqlBuffer.append(" and cityname = '"+pname+"'") ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) @@ -113,16 +114,19 @@ class ExportData { if(source && source=="高德"){ println index def type = rs.getObject("type") - def city = rs.getString("cityname") - if(citys.contains(city) && type.contains("商务住宅")){ + //def city = rs.getString("cityname") + if(type.contains("住宅")){ def ret = [] ret << city ret << rs.getString("adname") ret << rs.getString("name") ret << rs.getString("address") ret << rs.getString("type") - ret << rs.getFloat("location.lon") - ret << rs.getFloat("location.lat") + def loaction = rs.getObject("location") + if(loaction){ + ret << DataHandleUtils.getDoubleRound(6, loaction.x) + ret << DataHandleUtils.getDoubleRound(6, loaction.y) + } rets << ret } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 8eacf492b..ac634e9e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -68,13 +68,16 @@ class FangjiaClientUtils { public static def addressStand(def address){ def jsonObj = null + def content = null try { + address = address.replace("{","") def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") - def content = new HttpRequest().newGet(url, null, "utf-8") + content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) } catch (Exception e) { + println address +" "+content e.printStackTrace(); } return jsonObj; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy index 8f5809bed..f53276b72 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy @@ -1,9 +1,12 @@ package com.fangjia.script.gj.district +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DataUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation import java.util.regex.Pattern @@ -20,20 +23,13 @@ class MapDistrictProcess { * @return */ def run(){ - //"江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx","安徽地图.xlsx", - //"上海地图.xlsx","北京地图.xlsx" -//"安徽地图.xlsx","江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx", - //"重庆地图.xlsx","天津地图.xlsx", - //"上海地图.xlsx","北京地图.xlsx", - //"河北地图.xlsx","河南地图.xlsx","湖北地图.xlsx","湖南地图.xlsx","广西地图.xlsx", - //"吉林地图.xlsx","辽宁地图.xlsx""辽宁地图.xlsx","吉林地图.xlsx","黑龙江地图.xlsx","山西地图.xlsx","海南地图.xlsx"] - def files = ["青海地图.xlsx","宁夏地图.xlsx","内蒙古地图.xlsx","云南地图.xlsx","陕西地图.xlsx","新疆地图.xlsx","四川地图.xlsx"] + def files = ["武汉.xlsx"] for(def fileName : files){ def filePath = "D:\\数据\\小区\\地图\\"+fileName; - typeFilter(filePath) - nameStandard(filePath); - dataFilter(filePath) +// typeFilter(filePath) +// nameStandard(filePath); +// dataFilter(filePath) matchDistDatas(filePath) houseCalculate(filePath); ////// //统计是否存在多个区域 @@ -114,15 +110,24 @@ class MapDistrictProcess { def matchDistDatas(def path){ def index = 0 - ExcelUtils.multiThreadingModify(path,{ List row -> + ExcelUtils.modify(path,{ List row -> index ++ if(!row['I'] && !row['J']){ - def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],); + def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']); if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){ row['J'] = rMap.city row['K'] = rMap.region row['L'] = rMap.name row['M'] = rMap.address + + if(rMap.lat){ + def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3) + def maplng = baidu.x + def maplat = baidu.y + row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng) + def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L']) + row['Y'] = score + } } println index +" "+row } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 6a5c21a8e..5b0910fc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -26,15 +26,18 @@ class SeaweedDataMatchProcess { } } - if(!rMap && address){ -// if (findPattern(address)){ - rMap = OpenApiUtils.matchDistrict(city, address, region, true) -// } - } if(!rMap && name){ rMap = OpenApiUtils.matchDistrict(city, name, region, false) } + if(!rMap && address){ + rMap = OpenApiUtils.matchDistrict(city, address, region, true) + if(rMap && rMap.region && !region.contains(rMap.region)){ + rMap = null + } + } + + if(rMap && rMap.id && region.contains(rMap.region)) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy index e760b169f..cfa92f44a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy @@ -49,7 +49,7 @@ class ZsPersonProcess { // def loactionMap = task.matchMap(dist) // println loactionMap -// task.matchMap() + task.matchMap() // task.matchDb() // task.matchDb2() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index d8c7e72a3..be10cf533 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,9 +19,9 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() -// task.houseRun() + task.houseRun() - task.statHouseUpdateCount() +// task.statHouseUpdateCount() } def run(){ @@ -99,37 +99,35 @@ class EsMonthHouseStat { } def houseRun(){ - ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d-> - def city = d['B'] - if(city == "湖州"){ - def count = findHouseCount(city, 20210301, 20210601) - d['E'] = count + ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d-> + def city = d['A'] + def count = findHouseCount(city, 20190709, 20210709) + d['E'] = count - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); - def disthouseCount = 0 - for(def dist : districts){ - def alias = [] - if(dist.alias){ - if(!dist.alias.contains(dist.name)){ - dist.alias << dist.name - } - alias = dist.alias - }else{ - alias << dist.name + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def disthouseCount = 0 + for(def dist : districts){ + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name } - - def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601) - if(distCount){ - disthouseCount++ - } - println dist.city+" "+dist.region+" "+dist.name+" "+distCount + alias = dist.alias + }else{ + alias << dist.name } - println "disthouseCount:"+disthouseCount - println districts.size() - def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) - d['D'] = precent+"%" + def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709) + if(distCount){ + disthouseCount++ + } + println dist.city+" "+dist.region+" "+dist.name+" "+distCount } + + println "disthouseCount:"+disthouseCount + println districts.size() + def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) + d['F'] = precent+"%" },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy index c697657e8..a57c51cc7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy @@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{ @Override void process(Object args) { - citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""]) + citys.addAll(["福州"]) // getNoDataCitys(); def cityurl = "http://www.zhuge.com/areachange/" def cityUrls = getCityUrlCrawal(cityurl); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 6e5696107..841fb2f08 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -140,6 +140,7 @@ public class OpenApiClient { return null } + public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) { if (StringUtils.isBlank(input)) {