Merge remote-tracking branch 'origin/master'
This commit is contained in:
+1
-1
@@ -75,7 +75,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
|
||||
+3
-3
@@ -4660,7 +4660,7 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
q.put("city",new BasicDBObject('$in', ["深圳市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4694,7 +4694,7 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
||||
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
|
||||
q.put("city",new BasicDBObject('$in', ["深圳市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4755,7 +4755,7 @@ class CommonDatautils {
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
q.put("city",new BasicDBObject('$in', ["深圳"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
|
||||
@@ -3,7 +3,7 @@ package com.fangjia.script.gj.ccb
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FileUtils {
|
||||
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
|
||||
static def batchCreateFile(ret, fileName, headers, charset, isAppend){
|
||||
File file = new File(fileName);
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
@@ -51,6 +51,63 @@ class FileUtils {
|
||||
out.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def batchCreateFileByCustomSuffix(ret, fileName, headers, charset, isAppend, suffix, lineSuffix){
|
||||
File file = new File(fileName);
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
if(headers){
|
||||
sb.append(headers.join(suffix))
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), suffix))
|
||||
if(index<ret.size()){
|
||||
sb.append(lineSuffix+"\n");
|
||||
}else{
|
||||
sb.append(lineSuffix);
|
||||
}
|
||||
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def createTxtFile(ret, fileName, charset, isAppend){
|
||||
File file = new File(fileName);
|
||||
if (!file.getParentFile().exists()) {
|
||||
|
||||
+80
-23
@@ -17,7 +17,9 @@ import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.joda.time.Instant
|
||||
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DevProcess {
|
||||
@@ -35,11 +37,13 @@ class DevProcess {
|
||||
|
||||
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
|
||||
|
||||
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
task.devSplitRoad()
|
||||
// task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
@@ -56,7 +60,7 @@ class DevProcess {
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo100()
|
||||
task.synchDevTo100()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
@@ -75,10 +79,15 @@ class DevProcess {
|
||||
|
||||
// task.repeatSave()
|
||||
|
||||
// task.addDevInfo()
|
||||
|
||||
// task.matchDevShape()//匹配开发区围栏
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def parseCityAndDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
|
||||
if(!d['J']){
|
||||
@@ -271,30 +280,30 @@ class DevProcess {
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx"
|
||||
}
|
||||
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['C'] && !d['F']){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
||||
def devName = jsonObject1.get("dev")
|
||||
def obj = findDev(devName)
|
||||
d['F'] = devName
|
||||
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
|
||||
d['S'] = jsonObject1.detail.dev_confi
|
||||
}
|
||||
if (obj) {
|
||||
d['O'] = obj._id.toString()
|
||||
d['P'] = obj.name
|
||||
d['U'] = obj.match_way
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(d['C'] && !d['F']){
|
||||
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
// if(jsonObject1) {
|
||||
// if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
||||
// def devName = jsonObject1.get("dev")
|
||||
// def obj = findDev(devName)
|
||||
// d['F'] = devName
|
||||
// if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
|
||||
// d['S'] = jsonObject1.detail.dev_confi
|
||||
// }
|
||||
// if (obj) {
|
||||
// d['O'] = obj._id.toString()
|
||||
// d['P'] = obj.name
|
||||
// d['U'] = obj.match_way
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
if(d['E'] && !d['M']) {
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
@@ -326,7 +335,7 @@ class DevProcess {
|
||||
d['N'] = "信息不完整"
|
||||
}
|
||||
|
||||
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
|
||||
if (!d['N'] && !d['E'].contains(road)) {
|
||||
d['N'] = "原地址不含路"
|
||||
}
|
||||
|
||||
@@ -619,7 +628,7 @@ class DevProcess {
|
||||
|
||||
def addDevAddres3(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\河北省开发区地址入库.xlsx",{d->
|
||||
if(d['A']){
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
@@ -1063,4 +1072,52 @@ class DevProcess {
|
||||
}
|
||||
DEV_MAPPING_CONN2.insert(list)
|
||||
}
|
||||
|
||||
def addDevInfo(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\河北省开发区对应园区.xlsx",{d->
|
||||
if(d['C']){
|
||||
def q = new BasicDBObject().append("name", d['C'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['A'] = obj.city
|
||||
d['B'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchDevShape(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区-全量9-14.xlsx",{d->
|
||||
if(d['C'] && !d['I']){
|
||||
def newDev = d['E'].replace("经济技术开发区", "经济开发区").replace(d['B'].replace("省",""), "")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("city",d['C']).append("district", d['D']).append("type","乡镇街道").append("street", new BasicDBObject('$regex', Pattern.compile(newDev.trim()+'.*$', Pattern.CASE_INSENSITIVE))).append("shape", new BasicDBObject('$exists',true)))
|
||||
if(dbcursor.count()>0){
|
||||
println "[统计局] "+d['E'] +" "+ dbcursor.count()
|
||||
if(dbcursor.count()==1){
|
||||
for(def obj : dbcursor){
|
||||
d['F'] = obj.city
|
||||
d['G'] = obj.district
|
||||
d['H'] = obj.street
|
||||
d['I'] = "统计局相似(去省)-有围栏"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject("city",d['C']).append("region", d['D']).append("town", newDev))
|
||||
println "[乡镇街道围栏] "+d['E'] +" "+ dbcursor.count()
|
||||
if(dbcursor.count()==1){
|
||||
for(def obj : dbcursor){
|
||||
d['F'] = obj.city
|
||||
d['G'] = obj.region
|
||||
d['H'] = obj.town
|
||||
if(obj.polyline){
|
||||
d['I'] = "乡镇围栏相似(去省)-有围栏"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+13
-13
@@ -19,7 +19,7 @@ import org.elasticsearch.common.xcontent.XContentType
|
||||
class DevUpdateTestBatch {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
@@ -31,8 +31,8 @@ class DevUpdateTestBatch {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
// task.addDevAddress(path)
|
||||
|
||||
task.addDevAddress(path)
|
||||
//
|
||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ class DevUpdateTestBatch {
|
||||
def addDevAddress(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['D'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
@@ -140,15 +140,15 @@ class DevUpdateTestBatch {
|
||||
DEV_MAPPING_100_CONN.save(o)
|
||||
|
||||
|
||||
// o.id = o._id.toString()
|
||||
// o.remove("_id")
|
||||
// o.remove("c_date")
|
||||
// addEs(o, "192.168.0.89", "dev_mapping")
|
||||
// try{
|
||||
// addEs(o, "192.168.0.52", "dev_mapping")
|
||||
// }catch(Exception e){
|
||||
// println "52 es error: "+e.printStackTrace()
|
||||
// }
|
||||
o.id = o._id.toString()
|
||||
o.remove("_id")
|
||||
o.remove("c_date")
|
||||
addEs(o, "192.168.0.89", "dev_mapping")
|
||||
try{
|
||||
addEs(o, "192.168.0.52", "dev_mapping")
|
||||
}catch(Exception e){
|
||||
println "52 es error: "+e.printStackTrace()
|
||||
}
|
||||
|
||||
d['D'] = "已完成"
|
||||
}else{
|
||||
|
||||
+34
-8
@@ -21,11 +21,12 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
@@ -35,14 +36,14 @@ class PoiLocation {
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
// task.run();
|
||||
task.regionAndStreetProcess()
|
||||
task.run();
|
||||
// task.regionAndStreetProcess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = [""]
|
||||
def citys = ["深圳市"]
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
@@ -80,6 +81,8 @@ class PoiLocation {
|
||||
def findPois(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", "坪山区")
|
||||
q.put('address',new BasicDBObject('$regex', Pattern.compile('^.*兰竹东路8号.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
// q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
@@ -151,8 +154,8 @@ class PoiLocation {
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = running(d)
|
||||
@@ -170,8 +173,8 @@ class PoiLocation {
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
@@ -198,6 +201,8 @@ class PoiLocation {
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, d.city, d.township)
|
||||
}
|
||||
|
||||
def newTown = getTownByLocation(d.city, regions, d.location)
|
||||
if(town){
|
||||
if(town.street == d.township){
|
||||
//打标记
|
||||
@@ -311,6 +316,27 @@ class PoiLocation {
|
||||
return town
|
||||
}
|
||||
|
||||
def getTownByLocation(city,regions, location){
|
||||
def town = null
|
||||
def townFlag = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(city+"#"+r.district)
|
||||
for(def townObj : townList){
|
||||
if (townObj.shape && townObj.shape.coordinates) {
|
||||
townFlag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(townObj.shape.coordinates))
|
||||
if(townFlag){
|
||||
town = townObj
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if(townFlag){
|
||||
break
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id, def township){
|
||||
def header = [:]
|
||||
|
||||
+93
-6
@@ -1,7 +1,10 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -38,6 +41,9 @@ class TownMiddleProcess {
|
||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
@@ -57,7 +63,7 @@ class TownMiddleProcess {
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
synchTown()
|
||||
// synchTown()
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
@@ -70,7 +76,8 @@ class TownMiddleProcess {
|
||||
|
||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||
// updateCommunityDiffStreet()
|
||||
// updateTownPoline()
|
||||
// updateTownPoline()//更新乡镇街道围栏
|
||||
updateTownPolineBatch()//批量更新乡镇街道围栏
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
@@ -83,6 +90,9 @@ class TownMiddleProcess {
|
||||
// mzbDistrictUpate()
|
||||
|
||||
// addCity()
|
||||
|
||||
// addTownCenter()//百度接口添加中心点
|
||||
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
|
||||
}
|
||||
|
||||
|
||||
@@ -258,6 +268,7 @@ class TownMiddleProcess {
|
||||
|
||||
def createTown(){
|
||||
def cat = "乡镇街道"
|
||||
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def lastRets = []
|
||||
def govMap = findAllGovTown()
|
||||
@@ -853,9 +864,10 @@ class TownMiddleProcess {
|
||||
}
|
||||
|
||||
def updateTownPoline(){
|
||||
def city = "唐山市"
|
||||
def name = "高新技术开发区街道"
|
||||
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
|
||||
def city = "海口市"
|
||||
def region = "美兰区"
|
||||
def name = "海口桂林洋经济开发区(农场)"
|
||||
def q = new BasicDBObject("city", city).append("region", region).append("town","国营桂林洋农场")
|
||||
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
@@ -882,11 +894,51 @@ class TownMiddleProcess {
|
||||
udata.put("shape", resultMap)
|
||||
|
||||
|
||||
def updateQuery = new BasicDBObject("city", city).append("street",name)
|
||||
def updateQuery = new BasicDBObject("city", city).append("district", region).append("street",name)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def updateTownPolineBatch(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
|
||||
if(row['R'] && row['R']=="新增围栏"){
|
||||
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
|
||||
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
|
||||
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
|
||||
pointList << [lon, lat]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
|
||||
|
||||
def updateQuery = new BasicDBObject("name",row['N'])
|
||||
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
row['S'] = "已完成"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkTown(){
|
||||
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
@@ -1164,4 +1216,39 @@ class TownMiddleProcess {
|
||||
def updateQuery = new BasicDBObject("_id", id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
def addTownCenter(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道无中心点数据.xlsx",{ row->
|
||||
def json = BaiduMapUtils.searchByAddress(row['C']+row['D']+row['E'])
|
||||
if(json){
|
||||
row['J'] = json.confidence
|
||||
row['K'] = json.comprehension
|
||||
row['L'] = json.level
|
||||
|
||||
def location = json.location;
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(location.lng.toDouble(), location.lat.toDouble())
|
||||
if(gmap){
|
||||
gmap.lng = DataHandleUtils.getDoubleRound(6, gmap.lng)
|
||||
gmap.lat = DataHandleUtils.getDoubleRound(6, gmap.lat)
|
||||
|
||||
row['F'] = gmap.lng
|
||||
row['G'] = gmap.lat
|
||||
row['H'] = location.lng.toDouble()
|
||||
row['I'] = location.lat.toDouble()
|
||||
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkCenterInTownRang(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("location", new BasicDBObject('$exists',false))
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
|
||||
'address': '浙江省嘉兴市南湖区昌盛南路36号19幢嘉兴智慧产业创业园智慧大厦A座168室',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
|
||||
+1
-1
@@ -89,7 +89,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
sysnchVillage()
|
||||
// sysnchVillage()
|
||||
|
||||
// addTownship()
|
||||
}
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
/**
|
||||
* 厦门涉台园区数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
|
||||
|
||||
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
|
||||
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
|
||||
|
||||
DBCollection XM_RELATED_TW_QIYE_COLL = MongoUtils.getCollectionByName("business", "xm_related_tw_qiye", "mongo.db.fangjia.com")
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new RelatedTaiWanBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
createDatas() //企业库获取涉台园区企业
|
||||
updateRangeByPoint() //更改企业所在自贸区
|
||||
export() //导出数据
|
||||
}
|
||||
|
||||
|
||||
def updateRangeByPoint(){
|
||||
def zimaoqu_range_datas = []
|
||||
for(def s : zimaoqu_datas){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("name", s))
|
||||
for(def obj : dbcursor){
|
||||
if(obj && obj.shape){
|
||||
zimaoqu_range_datas << ["name":obj.name, "block":obj.block,"shape":obj.shape.coordinates]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
|
||||
println dbcursor.count()
|
||||
for(def r : dbcursor){
|
||||
def resKey = null
|
||||
for(def m : zimaoqu_range_datas){
|
||||
// def flag = InsidePolygon.isInArea(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
|
||||
def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
|
||||
if(flag){
|
||||
resKey = m.name
|
||||
break
|
||||
}
|
||||
}
|
||||
if(resKey){
|
||||
def udata = new BasicDBObject("free_trade_zone", resKey)
|
||||
def updateQuery = new BasicDBObject("_id",r._id)
|
||||
XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def createDatas(){
|
||||
def datas = findQiyeDatas()
|
||||
for(def r : datas){
|
||||
def map = getLocationInfo(r.city, r.address+r.company)
|
||||
if(map && map.location){
|
||||
def arrs = map.location.split(",")
|
||||
r.put("lon", DataHandleUtils.getDoubleRound(6, arrs[0].toString().toDouble()))
|
||||
r.put("lat", DataHandleUtils.getDoubleRound(6, arrs[1].toString().toDouble()))
|
||||
r.put("level", map.level)
|
||||
}
|
||||
saveToUpdate(r)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def saveToUpdate(datas){
|
||||
def udata = new BasicDBObject()
|
||||
datas.each{k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
XM_RELATED_TW_QIYE_COLL.save(udata)
|
||||
// def updateQuery = new BasicDBObject("name",row['N'])
|
||||
// XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
def findQiyeDatas(){
|
||||
def datas = []
|
||||
relateTaiwn_yuanqu_datas.each {
|
||||
def q = new BasicDBObject("name", it)
|
||||
def dbcursor = yuanqu_qiye_coll.find(q)
|
||||
for(def d : dbcursor){
|
||||
datas << ["_id":d._id, "province":d.province, "city":d.city,"address":d.address, "company":d.company, "name":d.name]
|
||||
}
|
||||
println it +" "+ datas.size()
|
||||
}
|
||||
return datas
|
||||
}
|
||||
|
||||
|
||||
def getLocationInfo(def city, def address){
|
||||
def gloJson = GaoDeApiUtils.geo(city, address)
|
||||
def map = [:]
|
||||
if(gloJson) {
|
||||
def glocation = null
|
||||
def level = null
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
glocation = geocodes.get(0).location
|
||||
level = geocodes.get(0).level
|
||||
}
|
||||
map.location = glocation
|
||||
map.level = level
|
||||
return map
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def export(){
|
||||
def rows = []
|
||||
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
|
||||
for(def r : dbcursor){
|
||||
rows << [r._id.toString(), r.province, r.city, r.address, r.company, r.name, r.free_trade_zone]
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat"
|
||||
}
|
||||
def allDatas = []
|
||||
allDatas.addAll(rows)
|
||||
// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
|
||||
FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName, curDate)
|
||||
|
||||
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
|
||||
}
|
||||
def rdy_datas = [rows.size().toString()]
|
||||
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName2, curDate)
|
||||
}
|
||||
}
|
||||
+10
-10
@@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
@@ -171,8 +171,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
@@ -262,8 +262,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
map.finish_build_time = dist.create_year?dist.create_year.toString():null
|
||||
if(dist.lng){
|
||||
def loc = [:]
|
||||
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
|
||||
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
|
||||
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
|
||||
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
|
||||
map.location = loc
|
||||
}
|
||||
|
||||
@@ -272,8 +272,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
@@ -281,8 +281,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
}
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jiangnanblank.SFTP
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import com.jcraft.jsch.SftpATTRS
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class XmBankFtpUtils {
|
||||
|
||||
static def ftpServer = "192.168.0.86"
|
||||
static def ftpPort = 17495
|
||||
static def ftpUser = "xmbank"
|
||||
static def ftpPassword = "Xmbank123!@#"
|
||||
static def ftpPath = "/output/company"
|
||||
|
||||
|
||||
|
||||
static SFTP sftp = new SFTP();
|
||||
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
|
||||
|
||||
static def ftpUpload(outputPath_dist, month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
sftp.upload(ftpPath, outputPath_dist, sftpChannel);
|
||||
|
||||
|
||||
}
|
||||
|
||||
static def reConnect(){
|
||||
if(sftpChannel == null){
|
||||
println "sftpChannel 为空,需重连....."
|
||||
if(sftp == null){
|
||||
sftp = new SFTP();
|
||||
}
|
||||
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
}else{
|
||||
println "sftpChannel 正常....."
|
||||
}
|
||||
return sftpChannel;
|
||||
}
|
||||
|
||||
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
|
||||
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
|
||||
|
||||
def data = new FileInputStream(new File(dataFileName))
|
||||
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def listFileNames(month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
|
||||
return files
|
||||
}
|
||||
|
||||
static def getFileLength(String path, String month){
|
||||
sftpChannel = reConnect();
|
||||
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
|
||||
return size
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+14
@@ -115,6 +115,20 @@ class InsidePolygon {
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
|
||||
public static List<Point> createPoint5(def pointList){
|
||||
List<Point> alllist = new ArrayList<>();
|
||||
for(def pointArr : pointList){
|
||||
Point point = new Point();
|
||||
double x = pointArr[0];
|
||||
double y = pointArr[1];
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
alllist.add(point);
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个经纬度点与区域的关系
|
||||
|
||||
+2
-1
@@ -99,7 +99,7 @@ class BaiduMapUtils {
|
||||
|
||||
public static JSONObject searchByAddress(String address) {
|
||||
String add = URLEncoder.encode(address, "utf-8");
|
||||
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=9E99D3540e4dafb7894f7b9fe27fa6b5"
|
||||
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=MZpy11ZrE8CbG4M5OznVAxIcRaBZ7aTR"
|
||||
boolean stop = false;
|
||||
int recount = 0;
|
||||
while(!stop && recount<3) {
|
||||
@@ -108,6 +108,7 @@ class BaiduMapUtils {
|
||||
try{
|
||||
String result = request.get(url,null,"UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(result);
|
||||
println address +" "+ json
|
||||
if(json.getInt("status") >= 300){
|
||||
changeAK();
|
||||
json = searchByAddress(address);
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ class GaoDeApiUtils {
|
||||
|
||||
def static geo(city, address) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", GAO_DE_AK)
|
||||
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//ÐíéF½Ü
|
||||
params.put("city", city)
|
||||
params.put("address", address)
|
||||
String content
|
||||
|
||||
Reference in New Issue
Block a user