Merge remote-tracking branch 'origin/master'
This commit is contained in:
+1
-1
@@ -75,7 +75,7 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
for(def city : allCitys){
|
for(def city : allCitys){
|
||||||
addDealPriceMatch(city);
|
addDealPriceMatch(city);
|
||||||
}
|
}
|
||||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||||
|
|
||||||
|
|
||||||
def rows = []
|
def rows = []
|
||||||
|
|||||||
+3
-3
@@ -4660,7 +4660,7 @@ class CommonDatautils {
|
|||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("type","区域")
|
q.put("type","区域")
|
||||||
q.put("district_diff",1)
|
q.put("district_diff",1)
|
||||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
q.put("city",new BasicDBObject('$in', ["深圳市"]))
|
||||||
|
|
||||||
def keys = new BasicDBObject()
|
def keys = new BasicDBObject()
|
||||||
keys.put("province",1)
|
keys.put("province",1)
|
||||||
@@ -4694,7 +4694,7 @@ class CommonDatautils {
|
|||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("type","乡镇街道")
|
q.put("type","乡镇街道")
|
||||||
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
||||||
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
|
q.put("city",new BasicDBObject('$in', ["深圳市"]))
|
||||||
|
|
||||||
def keys = new BasicDBObject()
|
def keys = new BasicDBObject()
|
||||||
keys.put("province",1)
|
keys.put("province",1)
|
||||||
@@ -4755,7 +4755,7 @@ class CommonDatautils {
|
|||||||
|
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("type","乡镇街道")
|
q.put("type","乡镇街道")
|
||||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
q.put("city",new BasicDBObject('$in', ["深圳"]))
|
||||||
|
|
||||||
def keys = new BasicDBObject()
|
def keys = new BasicDBObject()
|
||||||
keys.put("city",1)
|
keys.put("city",1)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package com.fangjia.script.gj.ccb
|
|||||||
import org.apache.commons.lang.StringUtils
|
import org.apache.commons.lang.StringUtils
|
||||||
|
|
||||||
class FileUtils {
|
class FileUtils {
|
||||||
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
|
static def batchCreateFile(ret, fileName, headers, charset, isAppend){
|
||||||
File file = new File(fileName);
|
File file = new File(fileName);
|
||||||
// if (file.exists()) {
|
// if (file.exists()) {
|
||||||
// file.delete();
|
// file.delete();
|
||||||
@@ -51,6 +51,63 @@ class FileUtils {
|
|||||||
out.close();
|
out.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
static def batchCreateFileByCustomSuffix(ret, fileName, headers, charset, isAppend, suffix, lineSuffix){
|
||||||
|
File file = new File(fileName);
|
||||||
|
// if (file.exists()) {
|
||||||
|
// file.delete();
|
||||||
|
// }
|
||||||
|
|
||||||
|
if (!file.getParentFile().exists()) {
|
||||||
|
boolean result = file.getParentFile().mkdirs();
|
||||||
|
if (!result) {
|
||||||
|
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||||
|
def sb = new StringBuffer();
|
||||||
|
|
||||||
|
if(headers){
|
||||||
|
sb.append(headers.join(suffix))
|
||||||
|
sb.append("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
out.write(sb.toString().getBytes(charset));
|
||||||
|
sb.setLength(0)
|
||||||
|
|
||||||
|
def index = 0
|
||||||
|
for(def row : ret){
|
||||||
|
if(row){
|
||||||
|
index++
|
||||||
|
sb.append(StringUtils.join(row.toArray(), suffix))
|
||||||
|
if(index<ret.size()){
|
||||||
|
sb.append(lineSuffix+"\n");
|
||||||
|
}else{
|
||||||
|
sb.append(lineSuffix);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(index%10000==0){
|
||||||
|
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||||
|
out.write(sb.toString().getBytes(charset));
|
||||||
|
sb.setLength(0)
|
||||||
|
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(sb.length()>0){
|
||||||
|
out.write(sb.toString().getBytes(charset));
|
||||||
|
sb.setLength(0)
|
||||||
|
}
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static def createTxtFile(ret, fileName, charset, isAppend){
|
static def createTxtFile(ret, fileName, charset, isAppend){
|
||||||
File file = new File(fileName);
|
File file = new File(fileName);
|
||||||
if (!file.getParentFile().exists()) {
|
if (!file.getParentFile().exists()) {
|
||||||
|
|||||||
+80
-23
@@ -17,7 +17,9 @@ import com.mongodb.DBCollection
|
|||||||
import net.sf.json.JSONObject
|
import net.sf.json.JSONObject
|
||||||
import org.apache.poi.ss.util.RegionUtil
|
import org.apache.poi.ss.util.RegionUtil
|
||||||
import org.bson.types.ObjectId
|
import org.bson.types.ObjectId
|
||||||
|
import org.joda.time.Instant
|
||||||
|
|
||||||
|
import java.time.format.DateTimeFormatter
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
class DevProcess {
|
class DevProcess {
|
||||||
@@ -35,11 +37,13 @@ class DevProcess {
|
|||||||
|
|
||||||
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
|
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
|
||||||
|
|
||||||
|
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new DevProcess();
|
def task = new DevProcess();
|
||||||
// task.run()
|
// task.run()
|
||||||
// task.parseCityAndDist()
|
// task.parseCityAndDist()
|
||||||
task.devSplitRoad()
|
// task.devSplitRoad()
|
||||||
// task.devNameSplitRoad()
|
// task.devNameSplitRoad()
|
||||||
// task.addFlag()
|
// task.addFlag()
|
||||||
// task.matchDev()
|
// task.matchDev()
|
||||||
@@ -56,7 +60,7 @@ class DevProcess {
|
|||||||
// task.createDevMapping()
|
// task.createDevMapping()
|
||||||
// task.deleteAddr()
|
// task.deleteAddr()
|
||||||
// task.deleteAddr2()
|
// task.deleteAddr2()
|
||||||
// task.synchDevTo100()
|
task.synchDevTo100()
|
||||||
// task.addNewDev2()
|
// task.addNewDev2()
|
||||||
|
|
||||||
// task.addAlias()
|
// task.addAlias()
|
||||||
@@ -75,10 +79,15 @@ class DevProcess {
|
|||||||
|
|
||||||
// task.repeatSave()
|
// task.repeatSave()
|
||||||
|
|
||||||
|
// task.addDevInfo()
|
||||||
|
|
||||||
|
// task.matchDevShape()//匹配开发区围栏
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def parseCityAndDist(){
|
def parseCityAndDist(){
|
||||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
|
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
|
||||||
if(!d['J']){
|
if(!d['J']){
|
||||||
@@ -271,30 +280,30 @@ class DevProcess {
|
|||||||
while(i<20){
|
while(i<20){
|
||||||
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
|
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
|
||||||
if(CommonDatautils.isWindow()){
|
if(CommonDatautils.isWindow()){
|
||||||
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
|
fileName = "D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx"
|
||||||
}
|
}
|
||||||
|
|
||||||
File file=new File(fileName);
|
File file=new File(fileName);
|
||||||
if(file.exists()){
|
if(file.exists()){
|
||||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||||
if(d['C'] && !d['F']){
|
// if(d['C'] && !d['F']){
|
||||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||||
if(jsonObject1) {
|
// if(jsonObject1) {
|
||||||
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
// if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
||||||
def devName = jsonObject1.get("dev")
|
// def devName = jsonObject1.get("dev")
|
||||||
def obj = findDev(devName)
|
// def obj = findDev(devName)
|
||||||
d['F'] = devName
|
// d['F'] = devName
|
||||||
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
|
// if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
|
||||||
d['S'] = jsonObject1.detail.dev_confi
|
// d['S'] = jsonObject1.detail.dev_confi
|
||||||
}
|
// }
|
||||||
if (obj) {
|
// if (obj) {
|
||||||
d['O'] = obj._id.toString()
|
// d['O'] = obj._id.toString()
|
||||||
d['P'] = obj.name
|
// d['P'] = obj.name
|
||||||
d['U'] = obj.match_way
|
// d['U'] = obj.match_way
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
if(d['E'] && !d['M']) {
|
if(d['E'] && !d['M']) {
|
||||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||||
@@ -326,7 +335,7 @@ class DevProcess {
|
|||||||
d['N'] = "信息不完整"
|
d['N'] = "信息不完整"
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
|
if (!d['N'] && !d['E'].contains(road)) {
|
||||||
d['N'] = "原地址不含路"
|
d['N'] = "原地址不含路"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -619,7 +628,7 @@ class DevProcess {
|
|||||||
|
|
||||||
def addDevAddres3(){
|
def addDevAddres3(){
|
||||||
def index = 0
|
def index = 0
|
||||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
|
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\河北省开发区地址入库.xlsx",{d->
|
||||||
if(d['A']){
|
if(d['A']){
|
||||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
|
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||||
if(obj){
|
if(obj){
|
||||||
@@ -1063,4 +1072,52 @@ class DevProcess {
|
|||||||
}
|
}
|
||||||
DEV_MAPPING_CONN2.insert(list)
|
DEV_MAPPING_CONN2.insert(list)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def addDevInfo(){
|
||||||
|
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\河北省开发区对应园区.xlsx",{d->
|
||||||
|
if(d['C']){
|
||||||
|
def q = new BasicDBObject().append("name", d['C'])
|
||||||
|
def obj = development_zone_CONN.findOne(q)
|
||||||
|
if(obj){
|
||||||
|
d['A'] = obj.city
|
||||||
|
d['B'] = obj.district
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def matchDevShape(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区-全量9-14.xlsx",{d->
|
||||||
|
if(d['C'] && !d['I']){
|
||||||
|
def newDev = d['E'].replace("经济技术开发区", "经济开发区").replace(d['B'].replace("省",""), "")
|
||||||
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("city",d['C']).append("district", d['D']).append("type","乡镇街道").append("street", new BasicDBObject('$regex', Pattern.compile(newDev.trim()+'.*$', Pattern.CASE_INSENSITIVE))).append("shape", new BasicDBObject('$exists',true)))
|
||||||
|
if(dbcursor.count()>0){
|
||||||
|
println "[统计局] "+d['E'] +" "+ dbcursor.count()
|
||||||
|
if(dbcursor.count()==1){
|
||||||
|
for(def obj : dbcursor){
|
||||||
|
d['F'] = obj.city
|
||||||
|
d['G'] = obj.district
|
||||||
|
d['H'] = obj.street
|
||||||
|
d['I'] = "统计局相似(去省)-有围栏"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject("city",d['C']).append("region", d['D']).append("town", newDev))
|
||||||
|
println "[乡镇街道围栏] "+d['E'] +" "+ dbcursor.count()
|
||||||
|
if(dbcursor.count()==1){
|
||||||
|
for(def obj : dbcursor){
|
||||||
|
d['F'] = obj.city
|
||||||
|
d['G'] = obj.region
|
||||||
|
d['H'] = obj.town
|
||||||
|
if(obj.polyline){
|
||||||
|
d['I'] = "乡镇围栏相似(去省)-有围栏"
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+13
-13
@@ -19,7 +19,7 @@ import org.elasticsearch.common.xcontent.XContentType
|
|||||||
class DevUpdateTestBatch {
|
class DevUpdateTestBatch {
|
||||||
|
|
||||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||||
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||||
|
|
||||||
|
|
||||||
@@ -31,8 +31,8 @@ class DevUpdateTestBatch {
|
|||||||
def task = new DevUpdateTestBatch()
|
def task = new DevUpdateTestBatch()
|
||||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||||
// task.addDev(path)
|
// task.addDev(path)
|
||||||
// task.addDevAddress(path)
|
task.addDevAddress(path)
|
||||||
|
//
|
||||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||||
|
|
||||||
|
|
||||||
@@ -109,7 +109,7 @@ class DevUpdateTestBatch {
|
|||||||
def addDevAddress(path){
|
def addDevAddress(path){
|
||||||
ExcelUtils.modify(path,{ d->
|
ExcelUtils.modify(path,{ d->
|
||||||
if(!d['D'] && d['A']){
|
if(!d['D'] && d['A']){
|
||||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||||
if(obj){
|
if(obj){
|
||||||
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
|
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
|
||||||
if(d['B'] == "乡镇街道"){
|
if(d['B'] == "乡镇街道"){
|
||||||
@@ -140,15 +140,15 @@ class DevUpdateTestBatch {
|
|||||||
DEV_MAPPING_100_CONN.save(o)
|
DEV_MAPPING_100_CONN.save(o)
|
||||||
|
|
||||||
|
|
||||||
// o.id = o._id.toString()
|
o.id = o._id.toString()
|
||||||
// o.remove("_id")
|
o.remove("_id")
|
||||||
// o.remove("c_date")
|
o.remove("c_date")
|
||||||
// addEs(o, "192.168.0.89", "dev_mapping")
|
addEs(o, "192.168.0.89", "dev_mapping")
|
||||||
// try{
|
try{
|
||||||
// addEs(o, "192.168.0.52", "dev_mapping")
|
addEs(o, "192.168.0.52", "dev_mapping")
|
||||||
// }catch(Exception e){
|
}catch(Exception e){
|
||||||
// println "52 es error: "+e.printStackTrace()
|
println "52 es error: "+e.printStackTrace()
|
||||||
// }
|
}
|
||||||
|
|
||||||
d['D'] = "已完成"
|
d['D'] = "已完成"
|
||||||
}else{
|
}else{
|
||||||
|
|||||||
+34
-8
@@ -21,11 +21,12 @@ import java.util.concurrent.CountDownLatch
|
|||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.atomic.AtomicLong
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
class PoiLocation {
|
class PoiLocation {
|
||||||
|
|
||||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
|
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||||
|
|
||||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||||
@@ -35,14 +36,14 @@ class PoiLocation {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new PoiLocation();
|
def task = new PoiLocation();
|
||||||
// task.run();
|
task.run();
|
||||||
task.regionAndStreetProcess()
|
// task.regionAndStreetProcess()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def citys = [""]
|
def citys = ["深圳市"]
|
||||||
citys = citys.reverse()
|
citys = citys.reverse()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
findPois(city)
|
findPois(city)
|
||||||
@@ -80,6 +81,8 @@ class PoiLocation {
|
|||||||
def findPois(city){
|
def findPois(city){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
|
q.put("district", "坪山区")
|
||||||
|
q.put('address',new BasicDBObject('$regex', Pattern.compile('^.*兰竹东路8号.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||||
// q.put("run_status", new BasicDBObject('$ne', 1))
|
// q.put("run_status", new BasicDBObject('$ne', 1))
|
||||||
|
|
||||||
def keys = new BasicDBObject()
|
def keys = new BasicDBObject()
|
||||||
@@ -151,8 +154,8 @@ class PoiLocation {
|
|||||||
for(def i=0;i<datas.size();i++){
|
for(def i=0;i<datas.size();i++){
|
||||||
final def d = datas[i]
|
final def d = datas[i]
|
||||||
|
|
||||||
fixedThreadPool.execute(new Runnable() {
|
// fixedThreadPool.execute(new Runnable() {
|
||||||
void run() {
|
// void run() {
|
||||||
try{
|
try{
|
||||||
totalCount.addAndGet(1)
|
totalCount.addAndGet(1)
|
||||||
def map = running(d)
|
def map = running(d)
|
||||||
@@ -170,8 +173,8 @@ class PoiLocation {
|
|||||||
} finally {
|
} finally {
|
||||||
latch.countDown()
|
latch.countDown()
|
||||||
}
|
}
|
||||||
}
|
// }
|
||||||
});
|
// });
|
||||||
}
|
}
|
||||||
latch.await()
|
latch.await()
|
||||||
return rets
|
return rets
|
||||||
@@ -198,6 +201,8 @@ class PoiLocation {
|
|||||||
if(d.township){
|
if(d.township){
|
||||||
town = getSimilarTown(regions, d.city, d.township)
|
town = getSimilarTown(regions, d.city, d.township)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def newTown = getTownByLocation(d.city, regions, d.location)
|
||||||
if(town){
|
if(town){
|
||||||
if(town.street == d.township){
|
if(town.street == d.township){
|
||||||
//打标记
|
//打标记
|
||||||
@@ -311,6 +316,27 @@ class PoiLocation {
|
|||||||
return town
|
return town
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def getTownByLocation(city,regions, location){
|
||||||
|
def town = null
|
||||||
|
def townFlag = null
|
||||||
|
for(def r : regions){
|
||||||
|
def townList = commMap.get(city+"#"+r.district)
|
||||||
|
for(def townObj : townList){
|
||||||
|
if (townObj.shape && townObj.shape.coordinates) {
|
||||||
|
townFlag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(townObj.shape.coordinates))
|
||||||
|
if(townFlag){
|
||||||
|
town = townObj
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(townFlag){
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return town
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def updateMapPoi(def id, def township){
|
def updateMapPoi(def id, def township){
|
||||||
def header = [:]
|
def header = [:]
|
||||||
|
|||||||
+93
-6
@@ -1,7 +1,10 @@
|
|||||||
package com.fangjia.script.gj.data_handle
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.core.util.GeocodeUtils
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.util.BaiduMapUtils
|
||||||
import com.fangjia.script.util.DataUtil
|
import com.fangjia.script.util.DataUtil
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
@@ -38,6 +41,9 @@ class TownMiddleProcess {
|
|||||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||||
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||||
|
|
||||||
|
|
||||||
|
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
@@ -57,7 +63,7 @@ class TownMiddleProcess {
|
|||||||
//修改乡镇街道别名
|
//修改乡镇街道别名
|
||||||
// updateTownAlias()
|
// updateTownAlias()
|
||||||
//中间表同步到100
|
//中间表同步到100
|
||||||
synchTown()
|
// synchTown()
|
||||||
//添加地级市标志
|
//添加地级市标志
|
||||||
// addCityFlag()
|
// addCityFlag()
|
||||||
// addPinyin()
|
// addPinyin()
|
||||||
@@ -70,7 +76,8 @@ class TownMiddleProcess {
|
|||||||
|
|
||||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||||
// updateCommunityDiffStreet()
|
// updateCommunityDiffStreet()
|
||||||
// updateTownPoline()
|
// updateTownPoline()//更新乡镇街道围栏
|
||||||
|
updateTownPolineBatch()//批量更新乡镇街道围栏
|
||||||
// checkTown()
|
// checkTown()
|
||||||
//区域民政部名称更新到乡镇街道中
|
//区域民政部名称更新到乡镇街道中
|
||||||
// updateTownshipByRegion()
|
// updateTownshipByRegion()
|
||||||
@@ -83,6 +90,9 @@ class TownMiddleProcess {
|
|||||||
// mzbDistrictUpate()
|
// mzbDistrictUpate()
|
||||||
|
|
||||||
// addCity()
|
// addCity()
|
||||||
|
|
||||||
|
// addTownCenter()//百度接口添加中心点
|
||||||
|
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -258,6 +268,7 @@ class TownMiddleProcess {
|
|||||||
|
|
||||||
def createTown(){
|
def createTown(){
|
||||||
def cat = "乡镇街道"
|
def cat = "乡镇街道"
|
||||||
|
|
||||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||||
def lastRets = []
|
def lastRets = []
|
||||||
def govMap = findAllGovTown()
|
def govMap = findAllGovTown()
|
||||||
@@ -853,9 +864,10 @@ class TownMiddleProcess {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def updateTownPoline(){
|
def updateTownPoline(){
|
||||||
def city = "唐山市"
|
def city = "海口市"
|
||||||
def name = "高新技术开发区街道"
|
def region = "美兰区"
|
||||||
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
|
def name = "海口桂林洋经济开发区(农场)"
|
||||||
|
def q = new BasicDBObject("city", city).append("region", region).append("town","国营桂林洋农场")
|
||||||
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
|
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
|
||||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||||
if(d){
|
if(d){
|
||||||
@@ -882,11 +894,51 @@ class TownMiddleProcess {
|
|||||||
udata.put("shape", resultMap)
|
udata.put("shape", resultMap)
|
||||||
|
|
||||||
|
|
||||||
def updateQuery = new BasicDBObject("city", city).append("street",name)
|
def updateQuery = new BasicDBObject("city", city).append("district", region).append("street",name)
|
||||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def updateTownPolineBatch(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
|
||||||
|
if(row['R'] && row['R']=="新增围栏"){
|
||||||
|
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
|
||||||
|
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
|
||||||
|
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||||
|
if(d){
|
||||||
|
def resultMap = [:]
|
||||||
|
resultMap.type = "polygon"
|
||||||
|
def arrs = d.polyline.split("\\|")
|
||||||
|
def rets = []
|
||||||
|
for(def s : arrs){
|
||||||
|
def points = s.split(";")
|
||||||
|
def pointList = []
|
||||||
|
for(def pointStr : points){
|
||||||
|
def pointArr = pointStr.split(",")
|
||||||
|
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
|
||||||
|
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
|
||||||
|
pointList << [lon, lat]
|
||||||
|
}
|
||||||
|
rets << pointList
|
||||||
|
}
|
||||||
|
resultMap.coordinates = rets
|
||||||
|
if(rets.size()>1) {
|
||||||
|
resultMap.type = "MultiPolygon"
|
||||||
|
}
|
||||||
|
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||||
|
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("shape", resultMap)
|
||||||
|
|
||||||
|
|
||||||
|
def updateQuery = new BasicDBObject("name",row['N'])
|
||||||
|
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||||
|
row['S'] = "已完成"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
def checkTown(){
|
def checkTown(){
|
||||||
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
|
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
|
||||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||||
@@ -1164,4 +1216,39 @@ class TownMiddleProcess {
|
|||||||
def updateQuery = new BasicDBObject("_id", id)
|
def updateQuery = new BasicDBObject("_id", id)
|
||||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def addTownCenter(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道无中心点数据.xlsx",{ row->
|
||||||
|
def json = BaiduMapUtils.searchByAddress(row['C']+row['D']+row['E'])
|
||||||
|
if(json){
|
||||||
|
row['J'] = json.confidence
|
||||||
|
row['K'] = json.comprehension
|
||||||
|
row['L'] = json.level
|
||||||
|
|
||||||
|
def location = json.location;
|
||||||
|
def gmap = GeocodeUtils.geoconvBaiduToGoogle(location.lng.toDouble(), location.lat.toDouble())
|
||||||
|
if(gmap){
|
||||||
|
gmap.lng = DataHandleUtils.getDoubleRound(6, gmap.lng)
|
||||||
|
gmap.lat = DataHandleUtils.getDoubleRound(6, gmap.lat)
|
||||||
|
|
||||||
|
row['F'] = gmap.lng
|
||||||
|
row['G'] = gmap.lat
|
||||||
|
row['H'] = location.lng.toDouble()
|
||||||
|
row['I'] = location.lat.toDouble()
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def checkCenterInTownRang(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("type","乡镇街道")
|
||||||
|
q.put("location", new BasicDBObject('$exists',false))
|
||||||
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||||
|
for(def d : dbcursor){
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -27,7 +27,7 @@ class QiliangTest {
|
|||||||
// task.run()
|
// task.run()
|
||||||
|
|
||||||
def data = [
|
def data = [
|
||||||
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
|
'address': '浙江省嘉兴市南湖区昌盛南路36号19幢嘉兴智慧产业创业园智慧大厦A座168室',
|
||||||
'type': '1' //1:对公 0:对私
|
'type': '1' //1:对公 0:对私
|
||||||
]
|
]
|
||||||
def s = task.getProxyResult(data);
|
def s = task.getProxyResult(data);
|
||||||
|
|||||||
+1
-1
@@ -89,7 +89,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
|||||||
// synchNetwork()
|
// synchNetwork()
|
||||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||||
// sysnchPoi()
|
// sysnchPoi()
|
||||||
sysnchVillage()
|
// sysnchVillage()
|
||||||
|
|
||||||
// addTownship()
|
// addTownship()
|
||||||
}
|
}
|
||||||
|
|||||||
+159
@@ -0,0 +1,159 @@
|
|||||||
|
package com.fangjia.script.gj.xiamen
|
||||||
|
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.BatchOwner
|
||||||
|
import com.fangjia.script.Watcher
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.gj.ccb.FileUtils
|
||||||
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.rabbitmq.client.AMQP
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 厦门涉台园区数据同步
|
||||||
|
*/
|
||||||
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
|
class RelatedTaiWanBatch extends BaseBatch{
|
||||||
|
|
||||||
|
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
|
||||||
|
|
||||||
|
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
|
||||||
|
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
|
||||||
|
|
||||||
|
DBCollection XM_RELATED_TW_QIYE_COLL = MongoUtils.getCollectionByName("business", "xm_related_tw_qiye", "mongo.db.fangjia.com")
|
||||||
|
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"]
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
runBatch(new RelatedTaiWanBatch(), args);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void process(Object args) {
|
||||||
|
createDatas() //企业库获取涉台园区企业
|
||||||
|
updateRangeByPoint() //更改企业所在自贸区
|
||||||
|
export() //导出数据
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def updateRangeByPoint(){
|
||||||
|
def zimaoqu_range_datas = []
|
||||||
|
for(def s : zimaoqu_datas){
|
||||||
|
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("name", s))
|
||||||
|
for(def obj : dbcursor){
|
||||||
|
if(obj && obj.shape){
|
||||||
|
zimaoqu_range_datas << ["name":obj.name, "block":obj.block,"shape":obj.shape.coordinates]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
|
||||||
|
println dbcursor.count()
|
||||||
|
for(def r : dbcursor){
|
||||||
|
def resKey = null
|
||||||
|
for(def m : zimaoqu_range_datas){
|
||||||
|
// def flag = InsidePolygon.isInArea(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
|
||||||
|
def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
|
||||||
|
if(flag){
|
||||||
|
resKey = m.name
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(resKey){
|
||||||
|
def udata = new BasicDBObject("free_trade_zone", resKey)
|
||||||
|
def updateQuery = new BasicDBObject("_id",r._id)
|
||||||
|
XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def createDatas(){
|
||||||
|
def datas = findQiyeDatas()
|
||||||
|
for(def r : datas){
|
||||||
|
def map = getLocationInfo(r.city, r.address+r.company)
|
||||||
|
if(map && map.location){
|
||||||
|
def arrs = map.location.split(",")
|
||||||
|
r.put("lon", DataHandleUtils.getDoubleRound(6, arrs[0].toString().toDouble()))
|
||||||
|
r.put("lat", DataHandleUtils.getDoubleRound(6, arrs[1].toString().toDouble()))
|
||||||
|
r.put("level", map.level)
|
||||||
|
}
|
||||||
|
saveToUpdate(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def saveToUpdate(datas){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
datas.each{k,v->
|
||||||
|
udata.put(k,v)
|
||||||
|
}
|
||||||
|
XM_RELATED_TW_QIYE_COLL.save(udata)
|
||||||
|
// def updateQuery = new BasicDBObject("name",row['N'])
|
||||||
|
// XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||||
|
}
|
||||||
|
|
||||||
|
def findQiyeDatas(){
|
||||||
|
def datas = []
|
||||||
|
relateTaiwn_yuanqu_datas.each {
|
||||||
|
def q = new BasicDBObject("name", it)
|
||||||
|
def dbcursor = yuanqu_qiye_coll.find(q)
|
||||||
|
for(def d : dbcursor){
|
||||||
|
datas << ["_id":d._id, "province":d.province, "city":d.city,"address":d.address, "company":d.company, "name":d.name]
|
||||||
|
}
|
||||||
|
println it +" "+ datas.size()
|
||||||
|
}
|
||||||
|
return datas
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getLocationInfo(def city, def address){
|
||||||
|
def gloJson = GaoDeApiUtils.geo(city, address)
|
||||||
|
def map = [:]
|
||||||
|
if(gloJson) {
|
||||||
|
def glocation = null
|
||||||
|
def level = null
|
||||||
|
if (gloJson && gloJson.get("geocodes")) {
|
||||||
|
def geocodes = gloJson.get("geocodes")
|
||||||
|
glocation = geocodes.get(0).location
|
||||||
|
level = geocodes.get(0).level
|
||||||
|
}
|
||||||
|
map.location = glocation
|
||||||
|
map.level = level
|
||||||
|
return map
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
def export(){
|
||||||
|
def rows = []
|
||||||
|
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
|
||||||
|
for(def r : dbcursor){
|
||||||
|
rows << [r._id.toString(), r.province, r.city, r.address, r.company, r.name, r.free_trade_zone]
|
||||||
|
}
|
||||||
|
|
||||||
|
def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat"
|
||||||
|
if(CommonDatautils.isWindow()){
|
||||||
|
fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat"
|
||||||
|
}
|
||||||
|
def allDatas = []
|
||||||
|
allDatas.addAll(rows)
|
||||||
|
// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
|
||||||
|
FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
|
||||||
|
XmBankFtpUtils.ftpUpload(fileName, curDate)
|
||||||
|
|
||||||
|
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
|
||||||
|
if(CommonDatautils.isWindow()){
|
||||||
|
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
|
||||||
|
}
|
||||||
|
def rdy_datas = [rows.size().toString()]
|
||||||
|
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
|
||||||
|
XmBankFtpUtils.ftpUpload(fileName2, curDate)
|
||||||
|
}
|
||||||
|
}
|
||||||
+10
-10
@@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
for(def i=0;i<dists.size();i++){
|
for(def i=0;i<dists.size();i++){
|
||||||
final def dist = dists[i]
|
final def dist = dists[i]
|
||||||
fixedThreadPool.execute(new Runnable() {
|
// fixedThreadPool.execute(new Runnable() {
|
||||||
void run() {
|
// void run() {
|
||||||
try{
|
try{
|
||||||
totalCount.addAndGet(1)
|
totalCount.addAndGet(1)
|
||||||
|
|
||||||
@@ -171,8 +171,8 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
} finally {
|
} finally {
|
||||||
latch.countDown()
|
latch.countDown()
|
||||||
}
|
}
|
||||||
}
|
// }
|
||||||
});
|
// });
|
||||||
}
|
}
|
||||||
latch.await()
|
latch.await()
|
||||||
return rets;
|
return rets;
|
||||||
@@ -262,8 +262,8 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
map.finish_build_time = dist.create_year?dist.create_year.toString():null
|
map.finish_build_time = dist.create_year?dist.create_year.toString():null
|
||||||
if(dist.lng){
|
if(dist.lng){
|
||||||
def loc = [:]
|
def loc = [:]
|
||||||
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
|
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
|
||||||
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
|
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
|
||||||
map.location = loc
|
map.location = loc
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -272,8 +272,8 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||||
if(baiduMap){
|
if(baiduMap){
|
||||||
def location = [:]
|
def location = [:]
|
||||||
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
|
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||||
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
|
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||||
map.bd_location = location
|
map.bd_location = location
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -281,8 +281,8 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||||
if(gpsLoactionMap){
|
if(gpsLoactionMap){
|
||||||
def gpslocation = [:]
|
def gpslocation = [:]
|
||||||
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
|
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||||
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
|
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||||
map.gps_location = gpslocation
|
map.gps_location = gpslocation
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+76
@@ -0,0 +1,76 @@
|
|||||||
|
package com.fangjia.script.gj.xiamen
|
||||||
|
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.jiangnanblank.SFTP
|
||||||
|
import com.jcraft.jsch.ChannelSftp
|
||||||
|
import com.jcraft.jsch.SftpATTRS
|
||||||
|
import org.apache.commons.codec.digest.DigestUtils
|
||||||
|
|
||||||
|
class XmBankFtpUtils {
|
||||||
|
|
||||||
|
static def ftpServer = "192.168.0.86"
|
||||||
|
static def ftpPort = 17495
|
||||||
|
static def ftpUser = "xmbank"
|
||||||
|
static def ftpPassword = "Xmbank123!@#"
|
||||||
|
static def ftpPath = "/output/company"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
static SFTP sftp = new SFTP();
|
||||||
|
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||||
|
|
||||||
|
|
||||||
|
static def ftpUpload(outputPath_dist, month){
|
||||||
|
//上传到ftp
|
||||||
|
sftpChannel = reConnect();
|
||||||
|
sftp.upload(ftpPath, outputPath_dist, sftpChannel);
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
static def reConnect(){
|
||||||
|
if(sftpChannel == null){
|
||||||
|
println "sftpChannel 为空,需重连....."
|
||||||
|
if(sftp == null){
|
||||||
|
sftp = new SFTP();
|
||||||
|
}
|
||||||
|
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||||
|
}else{
|
||||||
|
println "sftpChannel 正常....."
|
||||||
|
}
|
||||||
|
return sftpChannel;
|
||||||
|
}
|
||||||
|
|
||||||
|
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
|
||||||
|
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
|
||||||
|
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||||
|
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||||
|
}
|
||||||
|
|
||||||
|
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
|
||||||
|
|
||||||
|
def data = new FileInputStream(new File(dataFileName))
|
||||||
|
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
|
||||||
|
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||||
|
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||||
|
}
|
||||||
|
|
||||||
|
static def listFileNames(month){
|
||||||
|
//上传到ftp
|
||||||
|
sftpChannel = reConnect();
|
||||||
|
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
static def getFileLength(String path, String month){
|
||||||
|
sftpChannel = reConnect();
|
||||||
|
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
|
||||||
|
return size
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
+14
@@ -115,6 +115,20 @@ class InsidePolygon {
|
|||||||
}
|
}
|
||||||
return alllist
|
return alllist
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static List<Point> createPoint5(def pointList){
|
||||||
|
List<Point> alllist = new ArrayList<>();
|
||||||
|
for(def pointArr : pointList){
|
||||||
|
Point point = new Point();
|
||||||
|
double x = pointArr[0];
|
||||||
|
double y = pointArr[1];
|
||||||
|
point.setLon(x);
|
||||||
|
point.setLat(y);
|
||||||
|
alllist.add(point);
|
||||||
|
}
|
||||||
|
return alllist
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 判定一个经纬度点与区域的关系
|
* 判定一个经纬度点与区域的关系
|
||||||
|
|||||||
+2
-1
@@ -99,7 +99,7 @@ class BaiduMapUtils {
|
|||||||
|
|
||||||
public static JSONObject searchByAddress(String address) {
|
public static JSONObject searchByAddress(String address) {
|
||||||
String add = URLEncoder.encode(address, "utf-8");
|
String add = URLEncoder.encode(address, "utf-8");
|
||||||
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=9E99D3540e4dafb7894f7b9fe27fa6b5"
|
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=MZpy11ZrE8CbG4M5OznVAxIcRaBZ7aTR"
|
||||||
boolean stop = false;
|
boolean stop = false;
|
||||||
int recount = 0;
|
int recount = 0;
|
||||||
while(!stop && recount<3) {
|
while(!stop && recount<3) {
|
||||||
@@ -108,6 +108,7 @@ class BaiduMapUtils {
|
|||||||
try{
|
try{
|
||||||
String result = request.get(url,null,"UTF-8");
|
String result = request.get(url,null,"UTF-8");
|
||||||
JSONObject json = JSONObject.fromObject(result);
|
JSONObject json = JSONObject.fromObject(result);
|
||||||
|
println address +" "+ json
|
||||||
if(json.getInt("status") >= 300){
|
if(json.getInt("status") >= 300){
|
||||||
changeAK();
|
changeAK();
|
||||||
json = searchByAddress(address);
|
json = searchByAddress(address);
|
||||||
|
|||||||
+1
-1
@@ -40,7 +40,7 @@ class GaoDeApiUtils {
|
|||||||
|
|
||||||
def static geo(city, address) {
|
def static geo(city, address) {
|
||||||
Map<String, String> params = new HashMap<String, String>()
|
Map<String, String> params = new HashMap<String, String>()
|
||||||
params.put("key", GAO_DE_AK)
|
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//ÐíéF½Ü
|
||||||
params.put("city", city)
|
params.put("city", city)
|
||||||
params.put("address", address)
|
params.put("address", address)
|
||||||
String content
|
String content
|
||||||
|
|||||||
Reference in New Issue
Block a user