Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-09-24 15:11:13 +08:00
15 changed files with 546 additions and 69 deletions
@@ -75,7 +75,7 @@ class BigdataDealBatch extends BaseBatch{
for(def city : allCitys){
addDealPriceMatch(city);
}
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
@@ -4660,7 +4660,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
q.put("city",new BasicDBObject('$in', ["深圳市"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4694,7 +4694,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
q.put("city",new BasicDBObject('$in', ["深圳市"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4755,7 +4755,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
q.put("city",new BasicDBObject('$in', ["深圳"]))
def keys = new BasicDBObject()
keys.put("city",1)
@@ -3,7 +3,7 @@ package com.fangjia.script.gj.ccb
import org.apache.commons.lang.StringUtils
class FileUtils {
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
static def batchCreateFile(ret, fileName, headers, charset, isAppend){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
@@ -51,6 +51,63 @@ class FileUtils {
out.close();
}
static def batchCreateFileByCustomSuffix(ret, fileName, headers, charset, isAppend, suffix, lineSuffix){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
if(headers){
sb.append(headers.join(suffix))
sb.append("\n");
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), suffix))
if(index<ret.size()){
sb.append(lineSuffix+"\n");
}else{
sb.append(lineSuffix);
}
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
static def createTxtFile(ret, fileName, charset, isAppend){
File file = new File(fileName);
if (!file.getParentFile().exists()) {
@@ -17,7 +17,9 @@ import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.joda.time.Instant
import java.time.format.DateTimeFormatter
import java.util.regex.Pattern
class DevProcess {
@@ -35,11 +37,13 @@ class DevProcess {
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
static main(args) {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -56,7 +60,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo100()
task.synchDevTo100()
// task.addNewDev2()
// task.addAlias()
@@ -75,10 +79,15 @@ class DevProcess {
// task.repeatSave()
// task.addDevInfo()
// task.matchDevShape()//匹配开发区围栏
}
def parseCityAndDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
if(!d['J']){
@@ -271,30 +280,30 @@ class DevProcess {
while(i<20){
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
fileName = "D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['C'] && !d['F']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
if(jsonObject1) {
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
def devName = jsonObject1.get("dev")
def obj = findDev(devName)
d['F'] = devName
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
d['S'] = jsonObject1.detail.dev_confi
}
if (obj) {
d['O'] = obj._id.toString()
d['P'] = obj.name
d['U'] = obj.match_way
}
}
}
}
// if(d['C'] && !d['F']){
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
// if(jsonObject1) {
// if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
// def devName = jsonObject1.get("dev")
// def obj = findDev(devName)
// d['F'] = devName
// if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
// d['S'] = jsonObject1.detail.dev_confi
// }
// if (obj) {
// d['O'] = obj._id.toString()
// d['P'] = obj.name
// d['U'] = obj.match_way
// }
// }
// }
// }
if(d['E'] && !d['M']) {
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
@@ -326,7 +335,7 @@ class DevProcess {
d['N'] = "信息不完整"
}
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
if (!d['N'] && !d['E'].contains(road)) {
d['N'] = "原地址不含路"
}
@@ -619,7 +628,7 @@ class DevProcess {
def addDevAddres3(){
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\河北省开发区地址入库.xlsx",{d->
if(d['A']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
@@ -1063,4 +1072,52 @@ class DevProcess {
}
DEV_MAPPING_CONN2.insert(list)
}
def addDevInfo(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\河北省开发区对应园区.xlsx",{d->
if(d['C']){
def q = new BasicDBObject().append("name", d['C'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['A'] = obj.city
d['B'] = obj.district
}
}
})
}
def matchDevShape(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区-全量9-14.xlsx",{d->
if(d['C'] && !d['I']){
def newDev = d['E'].replace("经济技术开发区", "经济开发区").replace(d['B'].replace("省",""), "")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("city",d['C']).append("district", d['D']).append("type","乡镇街道").append("street", new BasicDBObject('$regex', Pattern.compile(newDev.trim()+'.*$', Pattern.CASE_INSENSITIVE))).append("shape", new BasicDBObject('$exists',true)))
if(dbcursor.count()>0){
println "[统计局] "+d['E'] +" "+ dbcursor.count()
if(dbcursor.count()==1){
for(def obj : dbcursor){
d['F'] = obj.city
d['G'] = obj.district
d['H'] = obj.street
d['I'] = "统计局相似(去省)-有围栏"
}
}
}else{
dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject("city",d['C']).append("region", d['D']).append("town", newDev))
println "[乡镇街道围栏] "+d['E'] +" "+ dbcursor.count()
if(dbcursor.count()==1){
for(def obj : dbcursor){
d['F'] = obj.city
d['G'] = obj.region
d['H'] = obj.town
if(obj.polyline){
d['I'] = "乡镇围栏相似(去省)-有围栏"
}
}
}
}
}
});
}
}
@@ -19,7 +19,7 @@ import org.elasticsearch.common.xcontent.XContentType
class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
@@ -31,8 +31,8 @@ class DevUpdateTestBatch {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
// task.addDevAddress(path)
task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -109,7 +109,7 @@ class DevUpdateTestBatch {
def addDevAddress(path){
ExcelUtils.modify(path,{ d->
if(!d['D'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
if(d['B'] == "乡镇街道"){
@@ -140,15 +140,15 @@ class DevUpdateTestBatch {
DEV_MAPPING_100_CONN.save(o)
// o.id = o._id.toString()
// o.remove("_id")
// o.remove("c_date")
// addEs(o, "192.168.0.89", "dev_mapping")
// try{
// addEs(o, "192.168.0.52", "dev_mapping")
// }catch(Exception e){
// println "52 es error: "+e.printStackTrace()
// }
o.id = o._id.toString()
o.remove("_id")
o.remove("c_date")
addEs(o, "192.168.0.89", "dev_mapping")
try{
addEs(o, "192.168.0.52", "dev_mapping")
}catch(Exception e){
println "52 es error: "+e.printStackTrace()
}
d['D'] = "已完成"
}else{
@@ -21,11 +21,12 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
@@ -35,14 +36,14 @@ class PoiLocation {
static main(args) {
def task = new PoiLocation();
// task.run();
task.regionAndStreetProcess()
task.run();
// task.regionAndStreetProcess()
}
def run(){
def citys = [""]
def citys = ["深圳市"]
citys = citys.reverse()
for(def city : citys){
findPois(city)
@@ -80,6 +81,8 @@ class PoiLocation {
def findPois(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", "坪山区")
q.put('address',new BasicDBObject('$regex', Pattern.compile('^.*兰竹东路8号.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
// q.put("run_status", new BasicDBObject('$ne', 1))
def keys = new BasicDBObject()
@@ -151,8 +154,8 @@ class PoiLocation {
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def map = running(d)
@@ -170,8 +173,8 @@ class PoiLocation {
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
return rets
@@ -198,6 +201,8 @@ class PoiLocation {
if(d.township){
town = getSimilarTown(regions, d.city, d.township)
}
def newTown = getTownByLocation(d.city, regions, d.location)
if(town){
if(town.street == d.township){
//打标记
@@ -311,6 +316,27 @@ class PoiLocation {
return town
}
def getTownByLocation(city,regions, location){
def town = null
def townFlag = null
for(def r : regions){
def townList = commMap.get(city+"#"+r.district)
for(def townObj : townList){
if (townObj.shape && townObj.shape.coordinates) {
townFlag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(townObj.shape.coordinates))
if(townFlag){
town = townObj
break
}
}
}
if(townFlag){
break
}
}
return town
}
def updateMapPoi(def id, def township){
def header = [:]
@@ -1,7 +1,10 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -38,6 +41,9 @@ class TownMiddleProcess {
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
@@ -57,7 +63,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
synchTown()
// synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -70,7 +76,8 @@ class TownMiddleProcess {
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet()
// updateTownPoline()
// updateTownPoline()//更新乡镇街道围栏
updateTownPolineBatch()//批量更新乡镇街道围栏
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
@@ -83,6 +90,9 @@ class TownMiddleProcess {
// mzbDistrictUpate()
// addCity()
// addTownCenter()//百度接口添加中心点
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
}
@@ -258,6 +268,7 @@ class TownMiddleProcess {
def createTown(){
def cat = "乡镇街道"
def tjjMaps = findAllProcincesFromTJJ(cat)
def lastRets = []
def govMap = findAllGovTown()
@@ -853,9 +864,10 @@ class TownMiddleProcess {
}
def updateTownPoline(){
def city = "唐山市"
def name = "高新技术开发区街道"
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
def city = "海口市"
def region = "美兰区"
def name = "海口桂林洋经济开发区(农场)"
def q = new BasicDBObject("city", city).append("region", region).append("town","国营桂林洋农场")
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
@@ -882,11 +894,51 @@ class TownMiddleProcess {
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("city", city).append("street",name)
def updateQuery = new BasicDBObject("city", city).append("district", region).append("street",name)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
def updateTownPolineBatch(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
if(row['R'] && row['R']=="新增围栏"){
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
pointList << [lon, lat]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
row['S'] = "已完成"
}
}
})
}
def checkTown(){
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
@@ -1164,4 +1216,39 @@ class TownMiddleProcess {
def updateQuery = new BasicDBObject("_id", id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
def addTownCenter(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道无中心点数据.xlsx",{ row->
def json = BaiduMapUtils.searchByAddress(row['C']+row['D']+row['E'])
if(json){
row['J'] = json.confidence
row['K'] = json.comprehension
row['L'] = json.level
def location = json.location;
def gmap = GeocodeUtils.geoconvBaiduToGoogle(location.lng.toDouble(), location.lat.toDouble())
if(gmap){
gmap.lng = DataHandleUtils.getDoubleRound(6, gmap.lng)
gmap.lat = DataHandleUtils.getDoubleRound(6, gmap.lat)
row['F'] = gmap.lng
row['G'] = gmap.lat
row['H'] = location.lng.toDouble()
row['I'] = location.lat.toDouble()
}
}
})
}
def checkCenterInTownRang(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("location", new BasicDBObject('$exists',false))
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
}
}
}
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
'address': '浙江省嘉兴市南湖区昌盛南路36号19幢嘉兴智慧产业创业园智慧大厦A座168室',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -89,7 +89,7 @@ class GaoDePoiDataProcess extends BaseBatch{
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
sysnchVillage()
// sysnchVillage()
// addTownship()
}
@@ -0,0 +1,159 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
/**
* 厦门涉台园区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class RelatedTaiWanBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
DBCollection XM_RELATED_TW_QIYE_COLL = MongoUtils.getCollectionByName("business", "xm_related_tw_qiye", "mongo.db.fangjia.com")
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"]
static main(args) {
runBatch(new RelatedTaiWanBatch(), args);
}
@Override
public void process(Object args) {
createDatas() //企业库获取涉台园区企业
updateRangeByPoint() //更改企业所在自贸区
export() //导出数据
}
def updateRangeByPoint(){
def zimaoqu_range_datas = []
for(def s : zimaoqu_datas){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("name", s))
for(def obj : dbcursor){
if(obj && obj.shape){
zimaoqu_range_datas << ["name":obj.name, "block":obj.block,"shape":obj.shape.coordinates]
}
}
}
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
println dbcursor.count()
for(def r : dbcursor){
def resKey = null
for(def m : zimaoqu_range_datas){
// def flag = InsidePolygon.isInArea(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
if(flag){
resKey = m.name
break
}
}
if(resKey){
def udata = new BasicDBObject("free_trade_zone", resKey)
def updateQuery = new BasicDBObject("_id",r._id)
XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
def createDatas(){
def datas = findQiyeDatas()
for(def r : datas){
def map = getLocationInfo(r.city, r.address+r.company)
if(map && map.location){
def arrs = map.location.split(",")
r.put("lon", DataHandleUtils.getDoubleRound(6, arrs[0].toString().toDouble()))
r.put("lat", DataHandleUtils.getDoubleRound(6, arrs[1].toString().toDouble()))
r.put("level", map.level)
}
saveToUpdate(r)
}
}
def saveToUpdate(datas){
def udata = new BasicDBObject()
datas.each{k,v->
udata.put(k,v)
}
XM_RELATED_TW_QIYE_COLL.save(udata)
// def updateQuery = new BasicDBObject("name",row['N'])
// XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
def findQiyeDatas(){
def datas = []
relateTaiwn_yuanqu_datas.each {
def q = new BasicDBObject("name", it)
def dbcursor = yuanqu_qiye_coll.find(q)
for(def d : dbcursor){
datas << ["_id":d._id, "province":d.province, "city":d.city,"address":d.address, "company":d.company, "name":d.name]
}
println it +" "+ datas.size()
}
return datas
}
def getLocationInfo(def city, def address){
def gloJson = GaoDeApiUtils.geo(city, address)
def map = [:]
if(gloJson) {
def glocation = null
def level = null
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
glocation = geocodes.get(0).location
level = geocodes.get(0).level
}
map.location = glocation
map.level = level
return map
}
return null
}
def export(){
def rows = []
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
for(def r : dbcursor){
rows << [r._id.toString(), r.province, r.city, r.address, r.company, r.name, r.free_trade_zone]
}
def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat"
}
def allDatas = []
allDatas.addAll(rows)
// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
XmBankFtpUtils.ftpUpload(fileName, curDate)
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
if(CommonDatautils.isWindow()){
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
}
def rdy_datas = [rows.size().toString()]
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
XmBankFtpUtils.ftpUpload(fileName2, curDate)
}
}
@@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
@@ -171,8 +171,8 @@ class SeaweedDataSynch extends BaseBatch{
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
return rets;
@@ -262,8 +262,8 @@ class SeaweedDataSynch extends BaseBatch{
map.finish_build_time = dist.create_year?dist.create_year.toString():null
if(dist.lng){
def loc = [:]
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
map.location = loc
}
@@ -272,8 +272,8 @@ class SeaweedDataSynch extends BaseBatch{
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
map.bd_location = location
}
@@ -281,8 +281,8 @@ class SeaweedDataSynch extends BaseBatch{
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
}
@@ -0,0 +1,76 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import com.jcraft.jsch.SftpATTRS
import org.apache.commons.codec.digest.DigestUtils
class XmBankFtpUtils {
static def ftpServer = "192.168.0.86"
static def ftpPort = 17495
static def ftpUser = "xmbank"
static def ftpPassword = "Xmbank123!@#"
static def ftpPath = "/output/company"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
static def ftpUpload(outputPath_dist, month){
//上传到ftp
sftpChannel = reConnect();
sftp.upload(ftpPath, outputPath_dist, sftpChannel);
}
static def reConnect(){
if(sftpChannel == null){
println "sftpChannel 为空,需重连....."
if(sftp == null){
sftp = new SFTP();
}
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
}else{
println "sftpChannel 正常....."
}
return sftpChannel;
}
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
def data = new FileInputStream(new File(dataFileName))
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def listFileNames(month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static def getFileLength(String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
return size
}
static main(args) {
}
}
@@ -115,6 +115,20 @@ class InsidePolygon {
}
return alllist
}
public static List<Point> createPoint5(def pointList){
List<Point> alllist = new ArrayList<>();
for(def pointArr : pointList){
Point point = new Point();
double x = pointArr[0];
double y = pointArr[1];
point.setLon(x);
point.setLat(y);
alllist.add(point);
}
return alllist
}
/**
* 判定一个经纬度点与区域的关系
@@ -99,7 +99,7 @@ class BaiduMapUtils {
public static JSONObject searchByAddress(String address) {
String add = URLEncoder.encode(address, "utf-8");
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=9E99D3540e4dafb7894f7b9fe27fa6b5"
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=MZpy11ZrE8CbG4M5OznVAxIcRaBZ7aTR"
boolean stop = false;
int recount = 0;
while(!stop && recount<3) {
@@ -108,6 +108,7 @@ class BaiduMapUtils {
try{
String result = request.get(url,null,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
println address +" "+ json
if(json.getInt("status") >= 300){
changeAK();
json = searchByAddress(address);
@@ -40,7 +40,7 @@ class GaoDeApiUtils {
def static geo(city, address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//ÐíéF½Ü
params.put("city", city)
params.put("address", address)
String content