This commit is contained in:
2023-08-26 13:34:26 +08:00
parent bf4e288396
commit a02526369a
16 changed files with 580 additions and 97 deletions
@@ -25,6 +25,7 @@ public class RegionData implements PriorityOrdered {
private final static Logger LOGGER = LoggerFactory.getLogger(RegionData.class);
private final static long REGION_RELOAD_INTERVAL = 1000;
private static DBCollection REGION_CONN = null;
private static DBCollection TOWN_CONN = null;
private static Set<String> ALL_CITY = new HashSet<String>();
private static Map<String,List<String>> cityAndAliasMap = new HashMap<String,List<String>>();
@@ -32,12 +33,15 @@ public class RegionData implements PriorityOrdered {
* 城市对应级别Map
*/
private static Map<String, Integer> cityGradeMap = new HashMap<String, Integer>();
protected final static Map<String, Map<String, Map<String, Map<String, List<String>>>>> townDataMap = new HashMap<String, Map<String, Map<String, Map<String, List<String>>>>>();
private static void init() {
try {
REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
TOWN_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1).append("alias", 1));
BasicDBObject dbo;
while(cursor.hasNext()){
@@ -58,6 +62,7 @@ public class RegionData implements PriorityOrdered {
start(REGION_RELOAD_INTERVAL);
reload();
// reloadTown();
}
private static void start(long time) {
@@ -114,6 +119,66 @@ public class RegionData implements PriorityOrdered {
LOGGER.info("RegionData reload() size [" + regionDataMap.size() + "]");
}
public static void reloadTown(){
DBObject query = new BasicDBObject();
query.put("year", 2022);
DBObject fields = new BasicDBObject();
fields.put("fangjia_city", 1);
fields.put("fangjia_area", 1);
fields.put("street_name", 1);
fields.put("alias", 1);
DBCursor cursor = TOWN_CONN.find(query, fields);
while (cursor.hasNext()) {
DBObject dbo = cursor.next();
String city = (String) dbo.get("fangjia_city");
String region = (String) dbo.get("fangjia_area");
String street = (String) dbo.get("street_name");
List<String> ps = (List<String>) dbo.get("alias");
if(!ALL_CITY.contains(city)){
continue;
}
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
if(cityRegionMap == null){
cityRegionMap = new HashMap<String, Map<String, Map<String, List<String>>>>();
townDataMap.put(city, cityRegionMap);
}
Map<String, Map<String, List<String>>> regoinMap = cityRegionMap.get(region);
if(regoinMap == null){
regoinMap = new HashMap<String, Map<String, List<String>>>();
cityRegionMap.put(region, regoinMap);
}
Map<String, List<String>> townMap = regoinMap.get("street");
if(townMap == null){
townMap = new HashMap<String, List<String>>();
regoinMap.put(street, townMap);
}
if (CollectionUtils.isNotEmpty(ps) && ps.size() > 1) {
Collections.sort(ps, new Comparator<String>() {
public int compare(String arg0, String arg1) {
if (StringUtils.isBlank(arg0)) {
return 1;
}
if (StringUtils.isBlank(arg1)) {
return -1;
}
return arg1.length() - arg0.length();
}
});
}
if(null != ps && !ps.contains(street)){
ps.add(street);
}
townMap.put(street, ps);
}
LOGGER.info("TownData reload() size [" + townDataMap.size() + "]");
}
static class ScheduleRunTask extends TimerTask {
private int nextScheduleHour;
@@ -174,6 +239,28 @@ public class RegionData implements PriorityOrdered {
return ret;
}
public static String formatStreet(String city, String region, String street) {
if (StringUtils.isBlank(region) || StringUtils.isBlank(city) || StringUtils.isBlank(street)) {
return null;
}
String ret = null;
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
if(MapUtils.isNotEmpty(cityRegionMap)){
Map<String, Map<String, List<String>>> streetMap = cityRegionMap.get(region);
if (MapUtils.isNotEmpty(streetMap)) {
for (Map<String, List<String>> crm : streetMap.values()) {
if (null != ret) {
break;
}
ret = format(crm, street);
}
}
}
return ret;
}
public static String parseRegionFormAddress(String city, String address){
if(StringUtils.isBlank(address)){
return null;
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -40,61 +40,61 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = null
if(today<=10){
ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
}else{
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
}
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
}
def allDatas = []
allDatas << headers
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
if(today > 10){
delPgData()
saveToPgSql(rows)
}
// def citys = CcbCommonUtils.getRunningCitys(args)
// def rows = []
//
// def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
// //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
// //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
//
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// def today = new Date().getDate()
//
// println "starting 。。。。。。。。。"
// for(def city : citys){
//
// index ++
//
//
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
//
//
// def ccbMaps = null
// if(today<=10){
// ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// }else{
// ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
// }
//
// def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
// rets = addDeleteDist(dists, ccbMaps, rets)
// rows.addAll(rets)
// println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
//
//// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
// }
//
// def allDatas = []
// allDatas << headers
// allDatas.addAll(rows)
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成->导Oracle"
//
// if(today > 10){
// delPgData()
// saveToPgSql(rows)
// }
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
delPgData()
saveToPgSql2(datas)
}
def addDeleteDist(dists, ccbMaps, rets){
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202306
def month = 202307
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202307)
def rets = findPeGteThirdParty2(city, 202308)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-04-01"
def date = "2023-05-01"
def endDate = "2023-06-01"
def newDate = "2023-07-01"
def startDate = "2023-05-01"
def date = "2023-06-01"
def endDate = "2023-07-01"
def newDate = "2023-08-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -3977,7 +3977,7 @@ class CommonDatautils {
}
static def findAmapDetailBySql(city){
String sql = "select * from amap_poi_detail where cityname='${city}'"
String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
// if(data.name.contains('华山小区G栋')){
@@ -3993,7 +3993,7 @@ class CommonDatautils {
def location = [:]
location.lon = point.x
location.lat = point.y
rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
// }
})
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-06-01"
static def statDate = "2023-07-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -1,16 +1,14 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.city
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
@@ -28,11 +26,25 @@ class CityStatExport {
static main(args) {
def task = new CityStatExport();
task.run()
// task.run()
// task.run1()
// task.regionStat()
// task.stat()
// task.houseStat()
task.addFieldStat()
}
def addFieldStat(){
ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d ->
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
if(cityInfoDto){
def detailMap = cityInfoDto.district_detail
d['D'] = detailMap.address.toDouble()*100
d['E'] = 100
d['F'] = cityInfoDto.price_coverage.toDouble()*100
}
})
}
def houseStat(){
@@ -60,7 +72,7 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity();
def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"]
def ret = []
ret << ["城市","小区总量"]
// ret << ["城市","楼栋总量", "房号总量"]
@@ -91,7 +103,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3")
}
@@ -25,8 +25,8 @@ class DistrictLocationBatch {
static main(args) {
def task = new DistrictLocationBatch()
// task.run()
task.deleteDist()
task.run()
// task.deleteDist()
}
def deleteDist(){
@@ -489,9 +489,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
if (citys.contains(e.text())){
// if (citys.contains(e.text())){
list << [e.text(), href]
}
// }
}
}
return list
@@ -65,7 +65,7 @@ public class LianjiaCrawlByChrome {
}
def run1(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -581,7 +581,7 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityList = []
def citys= ["福州"]
def citys= ["衢州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
@@ -0,0 +1,179 @@
package com.fangjia.script.gj.township
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class StreetProcess {
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new StreetProcess();
// println RegionData.formatRegion("鞍山","铁东区")
task.run()
}
def run(){
def region = RegionData.formatRegion("三明市", "三元区")
println region
// standard()
// matchStreet()
// synchTownship()
}
def synchTownship(){
def q = new BasicDBObject()
// q.put("city_name","福州市")
q.put("shape",new BasicDBObject('$exists', true))
def dbcursor = TOWNSHIP.find(q)
for(def d : dbcursor){
def udata = new BasicDBObject()
udata.put("shape", d.shape)
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
def matchStreet(){
def polylineMap = findStreetPolyline()
def q = new BasicDBObject()
// q.put("fangjia_city", "福州")
// q.put("shape",new BasicDBObject('$exists', false))
def dbcursor = TOWNSHIP.find(q)
def index = 0
for(def d : dbcursor){
index++
def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area)
if(streetMap){
def alias = d.alias
if(!alias){
alias = []
}
if(!alias.contains(d.street_name)){
alias << d.street_name
}
for(def streetName : alias){
def polyline = streetMap.get(streetName)
if(polyline){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
def udata = new BasicDBObject()
udata.put("shape", resultMap)
TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
println index + " "+ d.get("_id")
break;
}
}
}
}
}
def findStreetPolyline(){
def q = new BasicDBObject()
// q.put("fjw_city","福州")
def dbcursor = POLYLINE235.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.fjw_city+"##"+d.fjw_region
def streetMap = map.get(key)
if(!streetMap){
streetMap = [:]
map.put(key, streetMap)
}
streetMap.put(d.town, d.polyline)
}
return map;
}
def standard(){
def q = new BasicDBObject()
q.put("fjw_city",new BasicDBObject('$exists', false))
def dbcursor = POLYLINE235.find(q)
def index = 0
for(def d : dbcursor){
index ++
def city = RegionData.formatCity(d.city)
def region = RegionData.formatRegion(d.city, d.region)
// def street = null
// if(city && region){
// street = RegionData.formatStreet(city, region, d.town)
// }
def udata = new BasicDBObject()
udata.put("fjw_city", city)
udata.put("fjw_region", region)
// if(street){
// udata.put("fjw_street", street)
// }
POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println index
}
}
def standard1(){
def list = TOWNSHIP.find(new BasicDBObject()).toList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def city = RegionData.formatCity(d.city_name)
if(city){
def fangjia_province = ProvinceData.getProvince(city)
def region = RegionData.formatRegion(city, d.area_name)
def udata = new BasicDBObject()
udata.put("fangjia_city", city)
udata.put("fangjia_area", region)
udata.put("fangjia_province", fangjia_province)
TOWNSHIP.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println totalCount.get()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
}
}
@@ -6,8 +6,12 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -31,24 +35,66 @@ class GaoDePoiDataProcess extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static def version = new Date().getTime()
public static def TAGS = ["加油站","其它能源站","加气站","洗车场","充电站","餐饮相关场所","休闲餐饮场所","购物相关场所","商场","超级市场","家电电子卖场","花鸟鱼虫市场","家居建材市场","综合市场","特色商业街","生活服务场所","邮局","事务所","人才市场","洗浴推拿场所","体育休闲服务场所","运动场馆","娱乐场所","度假疗养场所","休闲场所","医疗保健服务场所","综合医院","专科医院","诊所","急救中心","疾病预防机构","住宿服务相关","宾馆酒店","旅馆招待所","风景名胜相关","公园广场","风景名胜","产业园区","政府及社会团体相关","政府机关","外国机构","社会团体","公检法机构","交通车辆管理","工商税务机构","科教文化场所","博物馆","展览馆","会展中心","美术馆","图书馆","科技馆","天文馆","文化宫","档案馆","文艺团体","传媒机构","学校","科研机构","培训机构","驾校","机场相关","银行","银行相关","保险公司","证券公司","财务公司","公司企业","知名企业","公司","工厂","农林牧渔基地","收费站","服务区","标志性建筑物","地名地址信息","热点地名","紧急避难场所"]
public final static DBCollection COMPANY_206_COLL = MongoUtils.getCollectionByName("business", "company", "192.168.0.206",27020)
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
static main(args) {
runBatch(new GaoDePoiDataProcess(), args)
}
@Override
void process(Object args) {
// synchMapPoi();
synchCompany()
}
def synchCompany(){
def citys = ["三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
for(def c : citys){
def townMap = findTownRange(c)
def companyDatas = findCompanyDatas(c)
println c+ " 加载公司数据:"+companyDatas.size()
running(companyDatas, "company", townMap)
println "完成"
}
}
def findCompanyDatas(city){
def q = new BasicDBObject()
q.put("done", 1)
q.put("geo_city", city)
q.put("geo_level", new BasicDBObject('$in', ['门牌号','兴趣点']))
return COMPANY_206_COLL.find(q).toList()
}
def synchMapPoi(){
//"三明市","漳州市","厦门市","福州市"
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"
def citys = ["福州市","重庆市"]
for(def city : citys){
def results = CommonDatautils.findAmapDetailBySql(city)
println results.size()
def rows = running(results)
def rows = running(results, "map")
save(city, rows)
}
}
def running(results){
def running(results, type, townMap){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
@@ -59,7 +105,11 @@ class GaoDePoiDataProcess extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
rets.addAll(createData(map))
if(type == "map"){
rets.addAll(createData(map, townMap))
}else if(type == "company"){
rets.addAll(createCompanyData(map, townMap))
}
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -73,7 +123,85 @@ class GaoDePoiDataProcess extends BaseBatch{
return rets;
}
def createData(map){
def createCompanyData(map, townMap){
def retMap = [:]
retMap.id = new ObjectId().toString()
retMap.fjw_province = map.province_name
retMap.fjw_city = map.city_name
def fjw_region = RegionData.formatRegion(map.city_name, map.geo_district)
if(fjw_region){
retMap.fjw_district = fjw_region
}
if(!retMap.fjw_district ){
retMap.fjw_district = RegionData.formatRegion(map.city_name, map.region_name)
}
retMap.province = map.geo_province
retMap.city = map.geo_city
retMap.district = map.geo_district
retMap.cat = "企业"
retMap.name = map.ent_name
// def pointList = townMap.get(retMap.fjw_district)
def locationStr = map.geo_location
if(locationStr){
def arrs = locationStr.split(",")
retMap.township = InsidePolygon.isInArea2(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), townMap)
def gd_location = [:]
gd_location.lon = arrs[0].toString().toDouble()
gd_location.lat = arrs[1].toString().toDouble()
retMap.location = gd_location
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
if(baiduMap){
def location = [:]
location.lon = baiduMap.lng
location.lat = baiduMap.lat
retMap.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = gpsLoactionMap.lng
gpslocation.lat = gpsLoactionMap.lat
retMap.gps_location = gpslocation
}
if(!retMap.township){
def gaodeMap = getTownshipInfo(locationStr)
if(gaodeMap && gaodeMap.township){
retMap.township = gaodeMap.township
}
if(!retMap.fjw_district){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
}
}
if(!fjw_region && !retMap.fjw_district){
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
}
retMap.fjw_district = fjw_region
}
}
retMap.address = map.reg_addr
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.ent_name, " ")
retMap.version = version
retMap.status = 1
return retMap
}
def createData(map, townMap){
def fjw_province = null
if(map.city.contains("重庆")){
fjw_province = "重庆"
@@ -83,7 +211,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
map.id = new ObjectId().toString()
map.fjw_province = fjw_province
map.fjw_province = fjw_province
map.fjw_city = fjw_city
if(map.cat){
@@ -128,8 +256,15 @@ class GaoDePoiDataProcess extends BaseBatch{
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
}
map.fjw_region = fjw_region
map.fjw_district = fjw_region
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
map.version = version
if(map.secondary_type && TAGS.contains(map.secondary_type)){
map.status = 0
}else{
map.status = 1
}
return map
}
@@ -140,7 +275,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def client = ElasticUtils.getClient("192.168.0.12")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest("amap_poi", "_doc", map.id).source(map, XContentType.JSON));
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
@@ -209,4 +344,27 @@ class GaoDePoiDataProcess extends BaseBatch{
return map;
}
def findTownRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
// def regionMap = townMap.get(d.fjw_region)
// if(!regionMap){
// regionMap = [:]
// townMap.put(d.fjw_region, regionMap)
// }
// def townInfoMap = regionMap.get(d.town)
// if(townInfoMap){
// townInfoMap = [:]
// regionMap.put()
// }
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
}
@@ -4,21 +4,32 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import com.fangjia.script.common.DataHandleUtils
class SeaweedDataSynch {
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
static main(args) {
def task = new SeaweedDataSynch()
task.run()
// println new ObjectId("622c433afb459ab08fdab460").getDate()
}
def run(){
@@ -46,9 +57,14 @@ class SeaweedDataSynch {
totalCount.addAndGet(1)
def map = createData(dist)
def newUrl = targetUrl +"_doc/"+map.id
CommonDatautils.httpSaveData(newUrl, map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
//往100 上同步
updateData(map)
//往12 es上同步
// def newUrl = targetUrl +"_doc/"+map.id
// CommonDatautils.httpSaveData(newUrl, map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
} catch (Exception e) {
e.printStackTrace()
@@ -63,9 +79,19 @@ class SeaweedDataSynch {
return rets;
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
if(k!="_id"){
udata.put(k,v)
}
}
SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
}
def createData(dist){
def map = [:]
map.id = dist._id.toString()
map._id = dist._id
map.city = dist.city
map.region = dist.region
map.name = dist.name
@@ -79,8 +105,8 @@ class SeaweedDataSynch {
map.township = dist.township
if(dist.lng){
def loc = [:]
loc.lon = dist.lng/10000000000.0
loc.lat = dist.lat/10000000000.0
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
map.location = loc
}
@@ -88,8 +114,8 @@ class SeaweedDataSynch {
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = baiduMap.lng
location.lat = baiduMap.lat
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
map.bd_location = location
}
@@ -97,8 +123,8 @@ class SeaweedDataSynch {
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = gpsLoactionMap.lng
gpslocation.lat = gpsLoactionMap.lat
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
@@ -272,4 +272,25 @@ class InsidePolygon {
}
return flag;
}
public static def isInArea2(lng, lat, pointMap){
def flag = false;
Set<String> keys = pointMap.keySet();
def name = null
for(String key :keys){
def prints = pointMap.get(key)
for(def pointArr : prints){
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
if(flag){
def arrs = key.split("##")
name = arrs[1];
break;
}
}
if(flag){
break;
}
}
return name;
}
}