diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java b/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java index d8b75d0c5..4add48eb1 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java @@ -25,6 +25,7 @@ public class RegionData implements PriorityOrdered { private final static Logger LOGGER = LoggerFactory.getLogger(RegionData.class); private final static long REGION_RELOAD_INTERVAL = 1000; private static DBCollection REGION_CONN = null; + private static DBCollection TOWN_CONN = null; private static Set ALL_CITY = new HashSet(); private static Map> cityAndAliasMap = new HashMap>(); @@ -32,12 +33,15 @@ public class RegionData implements PriorityOrdered { * 城市对应级别Map */ private static Map cityGradeMap = new HashMap(); + + protected final static Map>>>> townDataMap = new HashMap>>>>(); private static void init() { try { REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); - DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); + TOWN_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); + DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1).append("alias", 1)); BasicDBObject dbo; while(cursor.hasNext()){ @@ -58,6 +62,7 @@ public class RegionData implements PriorityOrdered { start(REGION_RELOAD_INTERVAL); reload(); +// reloadTown(); } private static void start(long time) { @@ -114,6 +119,66 @@ public class RegionData implements PriorityOrdered { LOGGER.info("RegionData reload() size [" + regionDataMap.size() + "]"); } + public static void reloadTown(){ + DBObject query = new BasicDBObject(); + query.put("year", 2022); + + DBObject fields = new BasicDBObject(); + fields.put("fangjia_city", 1); + fields.put("fangjia_area", 1); + fields.put("street_name", 1); + fields.put("alias", 1); + + + DBCursor cursor = TOWN_CONN.find(query, fields); + while (cursor.hasNext()) { + DBObject dbo = cursor.next(); + String city = (String) dbo.get("fangjia_city"); + String region = (String) dbo.get("fangjia_area"); + String street = (String) dbo.get("street_name"); + List ps = (List) dbo.get("alias"); + if(!ALL_CITY.contains(city)){ + continue; + } + Map>>> cityRegionMap = townDataMap.get(city); + if(cityRegionMap == null){ + cityRegionMap = new HashMap>>>(); + townDataMap.put(city, cityRegionMap); + } + + Map>> regoinMap = cityRegionMap.get(region); + if(regoinMap == null){ + regoinMap = new HashMap>>(); + cityRegionMap.put(region, regoinMap); + } + + Map> townMap = regoinMap.get("street"); + if(townMap == null){ + townMap = new HashMap>(); + regoinMap.put(street, townMap); + } + if (CollectionUtils.isNotEmpty(ps) && ps.size() > 1) { + Collections.sort(ps, new Comparator() { + public int compare(String arg0, String arg1) { + if (StringUtils.isBlank(arg0)) { + return 1; + } + if (StringUtils.isBlank(arg1)) { + return -1; + } + return arg1.length() - arg0.length(); + } + }); + } + if(null != ps && !ps.contains(street)){ + ps.add(street); + } + townMap.put(street, ps); + + } + + LOGGER.info("TownData reload() size [" + townDataMap.size() + "]"); + } static class ScheduleRunTask extends TimerTask { private int nextScheduleHour; @@ -174,6 +239,28 @@ public class RegionData implements PriorityOrdered { return ret; } + + public static String formatStreet(String city, String region, String street) { + if (StringUtils.isBlank(region) || StringUtils.isBlank(city) || StringUtils.isBlank(street)) { + return null; + } + + String ret = null; + Map>>> cityRegionMap = townDataMap.get(city); + if(MapUtils.isNotEmpty(cityRegionMap)){ + Map>> streetMap = cityRegionMap.get(region); + if (MapUtils.isNotEmpty(streetMap)) { + for (Map> crm : streetMap.values()) { + if (null != ret) { + break; + } + ret = format(crm, street); + } + } + } + return ret; + } + public static String parseRegionFormAddress(String city, String address){ if(StringUtils.isBlank(address)){ return null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 7ddc3a095..1453028a4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 49d4de707..7e64a374b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -40,61 +40,61 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) - def rows = [] - - def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", - //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", - //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] - - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 - def today = new Date().getDate() - - println "starting 。。。。。。。。。" - for(def city : citys){ - - index ++ - - - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - - - def ccbMaps = null - if(today<=10){ - ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 - }else{ - ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 - } - - def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) - rets = addDeleteDist(dists, ccbMaps, rets) - rows.addAll(rets) - println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() - -// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); - } - - def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" - } - - def allDatas = [] - allDatas << headers - allDatas.addAll(rows) - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成->导Oracle" - - if(today > 10){ - delPgData() - saveToPgSql(rows) - } +// def citys = CcbCommonUtils.getRunningCitys(args) +// def rows = [] +// +// def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", +// //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", +// //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] +// +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 +// def today = new Date().getDate() +// +// println "starting 。。。。。。。。。" +// for(def city : citys){ +// +// index ++ +// +// +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) +// +// +// def ccbMaps = null +// if(today<=10){ +// ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 +// }else{ +// ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 +// } +// +// def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) +// rets = addDeleteDist(dists, ccbMaps, rets) +// rows.addAll(rets) +// println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() +// +//// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" +// } +// +// def allDatas = [] +// allDatas << headers +// allDatas.addAll(rows) +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成->导Oracle" +// +// if(today > 10){ +// delPgData() +// saveToPgSql(rows) +// } // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") -// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") -// delPgData() -// saveToPgSql2(datas) + def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") + delPgData() + saveToPgSql2(datas) } def addDeleteDist(dists, ccbMaps, rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 87e5fa8d9..c2b5ce129 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -392,12 +392,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202306 + def month = 202307 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202307) + def rets = findPeGteThirdParty2(city, 202308) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 99dc7bf92..67ea43dda 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 71420fef7..4f338e082 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -93,13 +93,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-04-01" - def date = "2023-05-01" - def endDate = "2023-06-01" - def newDate = "2023-07-01" + def startDate = "2023-05-01" + def date = "2023-06-01" + def endDate = "2023-07-01" + def newDate = "2023-08-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index b81154fb8..c59cdcd87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3977,7 +3977,7 @@ class CommonDatautils { } static def findAmapDetailBySql(city){ - String sql = "select * from amap_poi_detail where cityname='${city}'" + String sql = "select * from amap_poi_detail where cityname='${city}' and status=0" def rets = [] ElasticUtils.executeSqlQuery(sql,{data-> // if(data.name.contains('华山小区G栋')){ @@ -3993,7 +3993,7 @@ class CommonDatautils { def location = [:] location.lon = point.x location.lat = point.y - rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] + rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] // } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index ac0ef538c..f06dd1590 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-06-01" + static def statDate = "2023-07-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy similarity index 91% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy index a7b5d4df1..4781397c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy @@ -1,16 +1,14 @@ -package com.fangjia.script.gj +package com.fangjia.script.gj.city import com.fangjia.core.RegionData import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils -import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject -import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators @@ -28,11 +26,25 @@ class CityStatExport { static main(args) { def task = new CityStatExport(); - task.run() +// task.run() // task.run1() // task.regionStat() // task.stat() // task.houseStat() + task.addFieldStat() + } + + def addFieldStat(){ + ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d -> + def cityInfoDto = CommonDatautils.findCityInfo(d['B']) + if(cityInfoDto){ + def detailMap = cityInfoDto.district_detail + d['D'] = detailMap.address.toDouble()*100 + d['E'] = 100 + d['F'] = cityInfoDto.price_coverage.toDouble()*100 + } + }) + } def houseStat(){ @@ -60,7 +72,7 @@ class CityStatExport { // def citys = CommonDatautils.findAllCity(); - def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] + def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"] def ret = [] ret << ["城市","小区总量"] // ret << ["城市","楼栋总量", "房号总量"] @@ -91,7 +103,7 @@ class CityStatExport { ret << m println m } - ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据") + ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy index c92e387fe..3c74690da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy @@ -25,8 +25,8 @@ class DistrictLocationBatch { static main(args) { def task = new DistrictLocationBatch() -// task.run() - task.deleteDist() + task.run() +// task.deleteDist() } def deleteDist(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 2003db863..8634731d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -489,9 +489,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") - if (citys.contains(e.text())){ +// if (citys.contains(e.text())){ list << [e.text(), href] - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index e43dddbb0..bcfef9581 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -65,7 +65,7 @@ public class LianjiaCrawlByChrome { } def run1(){ - System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() @@ -581,7 +581,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityList = [] - def citys= ["福州"] + def citys= ["衢州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy new file mode 100644 index 000000000..abb5bedc4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy @@ -0,0 +1,179 @@ +package com.fangjia.script.gj.township + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class StreetProcess { + + final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43") + + public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL) + + + final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + + static main(args) { + def task = new StreetProcess(); +// println RegionData.formatRegion("鞍山","铁东区") + task.run() + } + def run(){ + + def region = RegionData.formatRegion("三明市", "三元区") + println region + +// standard() +// matchStreet() +// synchTownship() + } + + def synchTownship(){ + def q = new BasicDBObject() +// q.put("city_name","福州市") + q.put("shape",new BasicDBObject('$exists', true)) + def dbcursor = TOWNSHIP.find(q) + for(def d : dbcursor){ + def udata = new BasicDBObject() + udata.put("shape", d.shape) + TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata)) + } + } + + def matchStreet(){ + def polylineMap = findStreetPolyline() + def q = new BasicDBObject() +// q.put("fangjia_city", "福州") +// q.put("shape",new BasicDBObject('$exists', false)) + def dbcursor = TOWNSHIP.find(q) + def index = 0 + for(def d : dbcursor){ + index++ + def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area) + if(streetMap){ + def alias = d.alias + if(!alias){ + alias = [] + } + if(!alias.contains(d.street_name)){ + alias << d.street_name + } + for(def streetName : alias){ + def polyline = streetMap.get(streetName) + if(polyline){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata)) + println index + " "+ d.get("_id") + break; + } + } + } + } + } + + def findStreetPolyline(){ + def q = new BasicDBObject() +// q.put("fjw_city","福州") + def dbcursor = POLYLINE235.find(q) + + def map = [:] + for(def d : dbcursor){ + def key = d.fjw_city+"##"+d.fjw_region + def streetMap = map.get(key) + if(!streetMap){ + streetMap = [:] + map.put(key, streetMap) + } + streetMap.put(d.town, d.polyline) + } + return map; + } + + + + + def standard(){ + def q = new BasicDBObject() + q.put("fjw_city",new BasicDBObject('$exists', false)) + def dbcursor = POLYLINE235.find(q) + def index = 0 + for(def d : dbcursor){ + index ++ + def city = RegionData.formatCity(d.city) + def region = RegionData.formatRegion(d.city, d.region) +// def street = null +// if(city && region){ +// street = RegionData.formatStreet(city, region, d.town) +// } + def udata = new BasicDBObject() + udata.put("fjw_city", city) + udata.put("fjw_region", region) +// if(street){ +// udata.put("fjw_street", street) +// } + POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println index + } + } + + def standard1(){ + def list = TOWNSHIP.find(new BasicDBObject()).toList() + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(list.size()) + for(def i=0;i + if(k!="_id"){ + udata.put(k,v) + } + } + SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false) + } + def createData(dist){ def map = [:] - map.id = dist._id.toString() + map._id = dist._id map.city = dist.city map.region = dist.region map.name = dist.name @@ -79,8 +105,8 @@ class SeaweedDataSynch { map.township = dist.township if(dist.lng){ def loc = [:] - loc.lon = dist.lng/10000000000.0 - loc.lat = dist.lat/10000000000.0 + loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0) + loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0) map.location = loc } @@ -88,8 +114,8 @@ class SeaweedDataSynch { def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon) if(baiduMap){ def location = [:] - location.lon = baiduMap.lng - location.lat = baiduMap.lat + location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng) + location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat) map.bd_location = location } @@ -97,8 +123,8 @@ class SeaweedDataSynch { def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon) if(gpsLoactionMap){ def gpslocation = [:] - gpslocation.lon = gpsLoactionMap.lng - gpslocation.lat = gpsLoactionMap.lat + gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng) + gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat) map.gps_location = gpslocation } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index 03f3e2b73..ac85c450c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -272,4 +272,25 @@ class InsidePolygon { } return flag; } + + public static def isInArea2(lng, lat, pointMap){ + def flag = false; + Set keys = pointMap.keySet(); + def name = null + for(String key :keys){ + def prints = pointMap.get(key) + for(def pointArr : prints){ + flag = InsidePolygon.isPtInPoly(lng, lat, pointArr) + if(flag){ + def arrs = key.split("##") + name = arrs[1]; + break; + } + } + if(flag){ + break; + } + } + return name; + } }