@@ -25,6 +25,7 @@ public class RegionData implements PriorityOrdered {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(RegionData.class);
|
||||
private final static long REGION_RELOAD_INTERVAL = 1000;
|
||||
private static DBCollection REGION_CONN = null;
|
||||
private static DBCollection TOWN_CONN = null;
|
||||
private static Set<String> ALL_CITY = new HashSet<String>();
|
||||
private static Map<String,List<String>> cityAndAliasMap = new HashMap<String,List<String>>();
|
||||
|
||||
@@ -32,12 +33,15 @@ public class RegionData implements PriorityOrdered {
|
||||
* 城市对应级别Map
|
||||
*/
|
||||
private static Map<String, Integer> cityGradeMap = new HashMap<String, Integer>();
|
||||
|
||||
protected final static Map<String, Map<String, Map<String, Map<String, List<String>>>>> townDataMap = new HashMap<String, Map<String, Map<String, Map<String, List<String>>>>>();
|
||||
|
||||
private static void init() {
|
||||
|
||||
try {
|
||||
REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
TOWN_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1).append("alias", 1));
|
||||
BasicDBObject dbo;
|
||||
while(cursor.hasNext()){
|
||||
@@ -58,6 +62,7 @@ public class RegionData implements PriorityOrdered {
|
||||
|
||||
start(REGION_RELOAD_INTERVAL);
|
||||
reload();
|
||||
// reloadTown();
|
||||
}
|
||||
|
||||
private static void start(long time) {
|
||||
@@ -114,6 +119,66 @@ public class RegionData implements PriorityOrdered {
|
||||
|
||||
LOGGER.info("RegionData reload() size [" + regionDataMap.size() + "]");
|
||||
}
|
||||
public static void reloadTown(){
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put("year", 2022);
|
||||
|
||||
DBObject fields = new BasicDBObject();
|
||||
fields.put("fangjia_city", 1);
|
||||
fields.put("fangjia_area", 1);
|
||||
fields.put("street_name", 1);
|
||||
fields.put("alias", 1);
|
||||
|
||||
|
||||
DBCursor cursor = TOWN_CONN.find(query, fields);
|
||||
while (cursor.hasNext()) {
|
||||
DBObject dbo = cursor.next();
|
||||
String city = (String) dbo.get("fangjia_city");
|
||||
String region = (String) dbo.get("fangjia_area");
|
||||
String street = (String) dbo.get("street_name");
|
||||
List<String> ps = (List<String>) dbo.get("alias");
|
||||
if(!ALL_CITY.contains(city)){
|
||||
continue;
|
||||
}
|
||||
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
|
||||
if(cityRegionMap == null){
|
||||
cityRegionMap = new HashMap<String, Map<String, Map<String, List<String>>>>();
|
||||
townDataMap.put(city, cityRegionMap);
|
||||
}
|
||||
|
||||
Map<String, Map<String, List<String>>> regoinMap = cityRegionMap.get(region);
|
||||
if(regoinMap == null){
|
||||
regoinMap = new HashMap<String, Map<String, List<String>>>();
|
||||
cityRegionMap.put(region, regoinMap);
|
||||
}
|
||||
|
||||
Map<String, List<String>> townMap = regoinMap.get("street");
|
||||
if(townMap == null){
|
||||
townMap = new HashMap<String, List<String>>();
|
||||
regoinMap.put(street, townMap);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(ps) && ps.size() > 1) {
|
||||
Collections.sort(ps, new Comparator<String>() {
|
||||
public int compare(String arg0, String arg1) {
|
||||
if (StringUtils.isBlank(arg0)) {
|
||||
return 1;
|
||||
}
|
||||
if (StringUtils.isBlank(arg1)) {
|
||||
return -1;
|
||||
}
|
||||
return arg1.length() - arg0.length();
|
||||
}
|
||||
});
|
||||
}
|
||||
if(null != ps && !ps.contains(street)){
|
||||
ps.add(street);
|
||||
}
|
||||
townMap.put(street, ps);
|
||||
|
||||
}
|
||||
|
||||
LOGGER.info("TownData reload() size [" + townDataMap.size() + "]");
|
||||
}
|
||||
|
||||
static class ScheduleRunTask extends TimerTask {
|
||||
private int nextScheduleHour;
|
||||
@@ -174,6 +239,28 @@ public class RegionData implements PriorityOrdered {
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public static String formatStreet(String city, String region, String street) {
|
||||
if (StringUtils.isBlank(region) || StringUtils.isBlank(city) || StringUtils.isBlank(street)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String ret = null;
|
||||
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
|
||||
if(MapUtils.isNotEmpty(cityRegionMap)){
|
||||
Map<String, Map<String, List<String>>> streetMap = cityRegionMap.get(region);
|
||||
if (MapUtils.isNotEmpty(streetMap)) {
|
||||
for (Map<String, List<String>> crm : streetMap.values()) {
|
||||
if (null != ret) {
|
||||
break;
|
||||
}
|
||||
ret = format(crm, street);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static String parseRegionFormAddress(String city, String address){
|
||||
if(StringUtils.isBlank(address)){
|
||||
return null;
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+54
-54
@@ -40,61 +40,61 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
|
||||
index ++
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
|
||||
def ccbMaps = null
|
||||
if(today<=10){
|
||||
ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
}else{
|
||||
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
}
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
allDatas << headers
|
||||
allDatas.addAll(rows)
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
// def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// def rows = []
|
||||
//
|
||||
// def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
// //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
// //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
//
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// def today = new Date().getDate()
|
||||
//
|
||||
// println "starting 。。。。。。。。。"
|
||||
// for(def city : citys){
|
||||
//
|
||||
// index ++
|
||||
//
|
||||
//
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
//
|
||||
//
|
||||
// def ccbMaps = null
|
||||
// if(today<=10){
|
||||
// ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// }else{
|
||||
// ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
// }
|
||||
//
|
||||
// def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
// rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
// rows.addAll(rets)
|
||||
// println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
//
|
||||
//// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// def allDatas = []
|
||||
// allDatas << headers
|
||||
// allDatas.addAll(rows)
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成->导Oracle"
|
||||
//
|
||||
// if(today > 10){
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
// }
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
// saveToPgSql2(datas)
|
||||
def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
delPgData()
|
||||
saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
|
||||
+2
-2
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202306
|
||||
def month = 202307
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202307)
|
||||
def rets = findPeGteThirdParty2(city, 202308)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+5
-5
@@ -93,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-04-01"
|
||||
def date = "2023-05-01"
|
||||
def endDate = "2023-06-01"
|
||||
def newDate = "2023-07-01"
|
||||
def startDate = "2023-05-01"
|
||||
def date = "2023-06-01"
|
||||
def endDate = "2023-07-01"
|
||||
def newDate = "2023-08-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+2
-2
@@ -3977,7 +3977,7 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
static def findAmapDetailBySql(city){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}'"
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
// if(data.name.contains('华山小区G栋')){
|
||||
@@ -3993,7 +3993,7 @@ class CommonDatautils {
|
||||
def location = [:]
|
||||
location.lon = point.x
|
||||
location.lat = point.y
|
||||
rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
|
||||
rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
|
||||
// }
|
||||
|
||||
})
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-06-01"
|
||||
static def statDate = "2023-07-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+18
-6
@@ -1,16 +1,14 @@
|
||||
package com.fangjia.script.gj
|
||||
package com.fangjia.script.gj.city
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
@@ -28,11 +26,25 @@ class CityStatExport {
|
||||
|
||||
static main(args) {
|
||||
def task = new CityStatExport();
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.regionStat()
|
||||
// task.stat()
|
||||
// task.houseStat()
|
||||
task.addFieldStat()
|
||||
}
|
||||
|
||||
def addFieldStat(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d ->
|
||||
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
|
||||
if(cityInfoDto){
|
||||
def detailMap = cityInfoDto.district_detail
|
||||
d['D'] = detailMap.address.toDouble()*100
|
||||
d['E'] = 100
|
||||
d['F'] = cityInfoDto.price_coverage.toDouble()*100
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def houseStat(){
|
||||
@@ -60,7 +72,7 @@ class CityStatExport {
|
||||
// def citys = CommonDatautils.findAllCity();
|
||||
|
||||
|
||||
def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
||||
def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"]
|
||||
def ret = []
|
||||
ret << ["城市","小区总量"]
|
||||
// ret << ["城市","楼栋总量", "房号总量"]
|
||||
@@ -91,7 +103,7 @@ class CityStatExport {
|
||||
ret << m
|
||||
println m
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据")
|
||||
ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3")
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -25,8 +25,8 @@ class DistrictLocationBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationBatch()
|
||||
// task.run()
|
||||
task.deleteDist()
|
||||
task.run()
|
||||
// task.deleteDist()
|
||||
}
|
||||
|
||||
def deleteDist(){
|
||||
|
||||
+2
-2
@@ -489,9 +489,9 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
if (citys.contains(e.text())){
|
||||
// if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+2
-2
@@ -65,7 +65,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run1(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -581,7 +581,7 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityList = []
|
||||
def citys= ["福州"]
|
||||
def citys= ["衢州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
package com.fangjia.script.gj.township
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class StreetProcess {
|
||||
|
||||
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
|
||||
|
||||
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
|
||||
|
||||
|
||||
final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new StreetProcess();
|
||||
// println RegionData.formatRegion("鞍山","铁东区")
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
def region = RegionData.formatRegion("三明市", "三元区")
|
||||
println region
|
||||
|
||||
// standard()
|
||||
// matchStreet()
|
||||
// synchTownship()
|
||||
}
|
||||
|
||||
def synchTownship(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city_name","福州市")
|
||||
q.put("shape",new BasicDBObject('$exists', true))
|
||||
def dbcursor = TOWNSHIP.find(q)
|
||||
for(def d : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", d.shape)
|
||||
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
def matchStreet(){
|
||||
def polylineMap = findStreetPolyline()
|
||||
def q = new BasicDBObject()
|
||||
// q.put("fangjia_city", "福州")
|
||||
// q.put("shape",new BasicDBObject('$exists', false))
|
||||
def dbcursor = TOWNSHIP.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area)
|
||||
if(streetMap){
|
||||
def alias = d.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(d.street_name)){
|
||||
alias << d.street_name
|
||||
}
|
||||
for(def streetName : alias){
|
||||
def polyline = streetMap.get(streetName)
|
||||
if(polyline){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
println index + " "+ d.get("_id")
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findStreetPolyline(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("fjw_city","福州")
|
||||
def dbcursor = POLYLINE235.find(q)
|
||||
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.fjw_city+"##"+d.fjw_region
|
||||
def streetMap = map.get(key)
|
||||
if(!streetMap){
|
||||
streetMap = [:]
|
||||
map.put(key, streetMap)
|
||||
}
|
||||
streetMap.put(d.town, d.polyline)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def standard(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fjw_city",new BasicDBObject('$exists', false))
|
||||
def dbcursor = POLYLINE235.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index ++
|
||||
def city = RegionData.formatCity(d.city)
|
||||
def region = RegionData.formatRegion(d.city, d.region)
|
||||
// def street = null
|
||||
// if(city && region){
|
||||
// street = RegionData.formatStreet(city, region, d.town)
|
||||
// }
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fjw_city", city)
|
||||
udata.put("fjw_region", region)
|
||||
// if(street){
|
||||
// udata.put("fjw_street", street)
|
||||
// }
|
||||
POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def standard1(){
|
||||
def list = TOWNSHIP.find(new BasicDBObject()).toList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def d = list[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def city = RegionData.formatCity(d.city_name)
|
||||
if(city){
|
||||
def fangjia_province = ProvinceData.getProvince(city)
|
||||
def region = RegionData.formatRegion(city, d.area_name)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fangjia_city", city)
|
||||
udata.put("fangjia_area", region)
|
||||
udata.put("fangjia_province", fangjia_province)
|
||||
TOWNSHIP.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println totalCount.get()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
}
|
||||
+165
-7
@@ -6,8 +6,12 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -31,24 +35,66 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static def version = new Date().getTime()
|
||||
|
||||
public static def TAGS = ["加油站","其它能源站","加气站","洗车场","充电站","餐饮相关场所","休闲餐饮场所","购物相关场所","商场","超级市场","家电电子卖场","花鸟鱼虫市场","家居建材市场","综合市场","特色商业街","生活服务场所","邮局","事务所","人才市场","洗浴推拿场所","体育休闲服务场所","运动场馆","娱乐场所","度假疗养场所","休闲场所","医疗保健服务场所","综合医院","专科医院","诊所","急救中心","疾病预防机构","住宿服务相关","宾馆酒店","旅馆招待所","风景名胜相关","公园广场","风景名胜","产业园区","政府及社会团体相关","政府机关","外国机构","社会团体","公检法机构","交通车辆管理","工商税务机构","科教文化场所","博物馆","展览馆","会展中心","美术馆","图书馆","科技馆","天文馆","文化宫","档案馆","文艺团体","传媒机构","学校","科研机构","培训机构","驾校","机场相关","银行","银行相关","保险公司","证券公司","财务公司","公司企业","知名企业","公司","工厂","农林牧渔基地","收费站","服务区","标志性建筑物","地名地址信息","热点地名","紧急避难场所"]
|
||||
|
||||
|
||||
public final static DBCollection COMPANY_206_COLL = MongoUtils.getCollectionByName("business", "company", "192.168.0.206",27020)
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
|
||||
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GaoDePoiDataProcess(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// synchMapPoi();
|
||||
synchCompany()
|
||||
}
|
||||
|
||||
def synchCompany(){
|
||||
|
||||
def citys = ["三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
|
||||
for(def c : citys){
|
||||
def townMap = findTownRange(c)
|
||||
def companyDatas = findCompanyDatas(c)
|
||||
println c+ " 加载公司数据:"+companyDatas.size()
|
||||
running(companyDatas, "company", townMap)
|
||||
println "完成"
|
||||
}
|
||||
}
|
||||
|
||||
def findCompanyDatas(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("done", 1)
|
||||
q.put("geo_city", city)
|
||||
q.put("geo_level", new BasicDBObject('$in', ['门牌号','兴趣点']))
|
||||
return COMPANY_206_COLL.find(q).toList()
|
||||
}
|
||||
|
||||
def synchMapPoi(){
|
||||
//"三明市","漳州市","厦门市","福州市"
|
||||
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"
|
||||
def citys = ["福州市","重庆市"]
|
||||
for(def city : citys){
|
||||
def results = CommonDatautils.findAmapDetailBySql(city)
|
||||
println results.size()
|
||||
def rows = running(results)
|
||||
def rows = running(results, "map")
|
||||
save(city, rows)
|
||||
}
|
||||
}
|
||||
|
||||
def running(results){
|
||||
|
||||
|
||||
def running(results, type, townMap){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
@@ -59,7 +105,11 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets.addAll(createData(map))
|
||||
if(type == "map"){
|
||||
rets.addAll(createData(map, townMap))
|
||||
}else if(type == "company"){
|
||||
rets.addAll(createCompanyData(map, townMap))
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -73,7 +123,85 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(map){
|
||||
def createCompanyData(map, townMap){
|
||||
def retMap = [:]
|
||||
retMap.id = new ObjectId().toString()
|
||||
retMap.fjw_province = map.province_name
|
||||
retMap.fjw_city = map.city_name
|
||||
|
||||
def fjw_region = RegionData.formatRegion(map.city_name, map.geo_district)
|
||||
if(fjw_region){
|
||||
retMap.fjw_district = fjw_region
|
||||
}
|
||||
|
||||
if(!retMap.fjw_district ){
|
||||
retMap.fjw_district = RegionData.formatRegion(map.city_name, map.region_name)
|
||||
}
|
||||
retMap.province = map.geo_province
|
||||
retMap.city = map.geo_city
|
||||
retMap.district = map.geo_district
|
||||
retMap.cat = "企业"
|
||||
retMap.name = map.ent_name
|
||||
// def pointList = townMap.get(retMap.fjw_district)
|
||||
|
||||
def locationStr = map.geo_location
|
||||
if(locationStr){
|
||||
def arrs = locationStr.split(",")
|
||||
|
||||
retMap.township = InsidePolygon.isInArea2(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), townMap)
|
||||
|
||||
def gd_location = [:]
|
||||
gd_location.lon = arrs[0].toString().toDouble()
|
||||
gd_location.lat = arrs[1].toString().toDouble()
|
||||
retMap.location = gd_location
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = baiduMap.lng
|
||||
location.lat = baiduMap.lat
|
||||
retMap.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = gpsLoactionMap.lng
|
||||
gpslocation.lat = gpsLoactionMap.lat
|
||||
retMap.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!retMap.township){
|
||||
def gaodeMap = getTownshipInfo(locationStr)
|
||||
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
retMap.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!retMap.fjw_district){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
if(!fjw_region && !retMap.fjw_district){
|
||||
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
|
||||
}
|
||||
retMap.fjw_district = fjw_region
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
retMap.address = map.reg_addr
|
||||
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.ent_name, " ")
|
||||
retMap.version = version
|
||||
retMap.status = 1
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def createData(map, townMap){
|
||||
def fjw_province = null
|
||||
if(map.city.contains("重庆")){
|
||||
fjw_province = "重庆"
|
||||
@@ -83,7 +211,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def fjw_city = RegionData.formatCity(map.city)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
|
||||
map.id = new ObjectId().toString()
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_city = fjw_city
|
||||
|
||||
if(map.cat){
|
||||
@@ -128,8 +256,15 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
}
|
||||
map.fjw_region = fjw_region
|
||||
map.fjw_district = fjw_region
|
||||
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||
map.version = version
|
||||
|
||||
if(map.secondary_type && TAGS.contains(map.secondary_type)){
|
||||
map.status = 0
|
||||
}else{
|
||||
map.status = 1
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
@@ -140,7 +275,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def client = ElasticUtils.getClient("192.168.0.12")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest("amap_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
@@ -209,4 +344,27 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
return map;
|
||||
}
|
||||
|
||||
def findTownRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
// def regionMap = townMap.get(d.fjw_region)
|
||||
// if(!regionMap){
|
||||
// regionMap = [:]
|
||||
// townMap.put(d.fjw_region, regionMap)
|
||||
// }
|
||||
// def townInfoMap = regionMap.get(d.town)
|
||||
// if(townInfoMap){
|
||||
// townInfoMap = [:]
|
||||
// regionMap.put()
|
||||
// }
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+36
-10
@@ -4,21 +4,32 @@ import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
class SeaweedDataSynch {
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new SeaweedDataSynch()
|
||||
task.run()
|
||||
// println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -46,9 +57,14 @@ class SeaweedDataSynch {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def map = createData(dist)
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
CommonDatautils.httpSaveData(newUrl, map)
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
||||
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
|
||||
//往12 es上同步
|
||||
// def newUrl = targetUrl +"_doc/"+map.id
|
||||
// CommonDatautils.httpSaveData(newUrl, map)
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
@@ -63,9 +79,19 @@ class SeaweedDataSynch {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def updateData(map){
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(k!="_id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map.id = dist._id.toString()
|
||||
map._id = dist._id
|
||||
map.city = dist.city
|
||||
map.region = dist.region
|
||||
map.name = dist.name
|
||||
@@ -79,8 +105,8 @@ class SeaweedDataSynch {
|
||||
map.township = dist.township
|
||||
if(dist.lng){
|
||||
def loc = [:]
|
||||
loc.lon = dist.lng/10000000000.0
|
||||
loc.lat = dist.lat/10000000000.0
|
||||
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
|
||||
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
|
||||
map.location = loc
|
||||
}
|
||||
|
||||
@@ -88,8 +114,8 @@ class SeaweedDataSynch {
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = baiduMap.lng
|
||||
location.lat = baiduMap.lat
|
||||
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
@@ -97,8 +123,8 @@ class SeaweedDataSynch {
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = gpsLoactionMap.lng
|
||||
gpslocation.lat = gpsLoactionMap.lat
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
|
||||
+21
@@ -272,4 +272,25 @@ class InsidePolygon {
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
public static def isInArea2(lng, lat, pointMap){
|
||||
def flag = false;
|
||||
Set<String> keys = pointMap.keySet();
|
||||
def name = null
|
||||
for(String key :keys){
|
||||
def prints = pointMap.get(key)
|
||||
for(def pointArr : prints){
|
||||
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
|
||||
if(flag){
|
||||
def arrs = key.split("##")
|
||||
name = arrs[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user