update
This commit is contained in:
@@ -531,6 +531,22 @@
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.aliyun</groupId>
|
||||
<artifactId>aliyun-java-sdk-address-purification</artifactId>
|
||||
<version>1.0.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.aliyun</groupId>
|
||||
<artifactId>aliyun-java-sdk-core</artifactId>
|
||||
<version>4.5.9</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
<version>1.2.83</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+5
-6
@@ -28,7 +28,8 @@ import org.bson.types.ObjectId
|
||||
class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
|
||||
|
||||
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
|
||||
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
|
||||
@@ -215,7 +216,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("city", city)
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
query.put("status", 0)
|
||||
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
@@ -227,11 +228,9 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
int tryTimes = 5
|
||||
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
|
||||
for(int i=0; i < tryTimes; i++) {
|
||||
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
|
||||
def cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
|
||||
int totalCount = cur.count()
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
while(iter.hasNext()) {
|
||||
BasicDBObject dbo = (BasicDBObject)iter.next()
|
||||
for(def dbo : cur) {
|
||||
districts.add(dbo)
|
||||
}
|
||||
if(districts.size() == totalCount) {
|
||||
|
||||
+1
-1
@@ -4007,7 +4007,7 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
static def findAmapDetailBySql(city){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
|
||||
String sql = "select * from amap_poi where cityname='${city}' and status=0"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
// if(data.name.contains('华山小区G栋')){
|
||||
|
||||
+20
@@ -348,4 +348,24 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def villageIdentify(city, address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify?address="+URLEncoder.encode(address,"UTF-8")+"&type=2"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+345
@@ -0,0 +1,345 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
def filePath = "D:\\数据\\光大\\全量.xlsx"
|
||||
//ExcelUtils.modify(filePath, { row ->
|
||||
// if(!row['O']){
|
||||
// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A'])
|
||||
// if(jsonObj && jsonObj.result){
|
||||
// row['B'] = jsonObj.result.countyLevelCity
|
||||
// if(row['B']){
|
||||
// if(row['B']=="是"){
|
||||
// row['C'] = "否"
|
||||
// }else if(row['B']=="否"){
|
||||
// row['C'] = "是"
|
||||
// }
|
||||
// }
|
||||
// row['D'] = jsonObj.result.district
|
||||
// row['E'] = jsonObj.result.districtCode
|
||||
// if(jsonObj.result.administrativeTowns && jsonObj.result.administrativeTowns=="是"){
|
||||
// row['F'] = jsonObj.result.administrativeTowns
|
||||
// row['G'] = jsonObj.result.town
|
||||
// row['H'] = jsonObj.result.townCode
|
||||
// }
|
||||
//
|
||||
// if(jsonObj.result.countyGov && jsonObj.result.countyGov=="是"){
|
||||
// row['I'] = jsonObj.result.countyGov
|
||||
// row['J'] = jsonObj.result.town
|
||||
// row['K'] = jsonObj.result.townCode
|
||||
// }
|
||||
//
|
||||
// if(jsonObj.result.devDistrict && jsonObj.result.devDistrict=="是"){
|
||||
// row['L'] = jsonObj.result.devDistrict
|
||||
// row['M'] = jsonObj.result.devDistrictName
|
||||
// }
|
||||
// row['N'] = jsonObj.result.addrVerify
|
||||
// row['O'] = jsonObj.result.province
|
||||
// row['P'] = jsonObj.result.city
|
||||
// row['Q'] = jsonObj.result.committee
|
||||
// row['R'] = jsonObj.result.typeName
|
||||
// row['S'] = jsonObj.result.splitCommittee
|
||||
// row['T'] = jsonObj.result.splitStreet
|
||||
// row['U'] = jsonObj.result.splitStreetNum
|
||||
// row['V'] = jsonObj.result.poiName
|
||||
// row['W'] = jsonObj.result.dev
|
||||
// }
|
||||
// }
|
||||
|
||||
// def retMap = SignatureUtils.getBaiduAddressStrc(row['A'])
|
||||
// println row['A'] +" " + retMap.toString()
|
||||
// if(retMap){
|
||||
// row['Y'] = retMap.prov
|
||||
// row['Z'] = retMap.city
|
||||
// row['AA'] = retMap.district
|
||||
// row['AB'] = retMap.town
|
||||
// row['AC'] = retMap.devzone
|
||||
// row['AD'] = retMap.road
|
||||
// row['AE'] = retMap.roadno
|
||||
// row['AF'] = retMap.poi
|
||||
// row['AG'] = retMap.subpoi
|
||||
// }
|
||||
// })
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
//
|
||||
// if(row['AA'] && !row['AO']){
|
||||
// if(row['Y']){
|
||||
// if(row['Y'].endsWith("区")){
|
||||
// row['AE'] = "否"
|
||||
// row['AF'] = "是"
|
||||
// }else {
|
||||
// row['AE'] = "是"
|
||||
// row['AF'] = "否"
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(row['AA']){
|
||||
// def dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], row['AA'])
|
||||
// if(!dev_name && row['AA'].contains("经济开发区")){
|
||||
// dev_name = row['AA'].replace("经济开发区","经济技术开发区")
|
||||
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
|
||||
// }
|
||||
// if(!dev_name && row['AA'].contains("经济技术开发区")){
|
||||
// dev_name = row['AA'].replace("经济技术开发区","经济开发区")
|
||||
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
|
||||
// }
|
||||
// if(dev_name){
|
||||
// row['AO'] = "是"
|
||||
// row['AP'] = dev_name
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(row['W'] && row['X'] && row['Y'] && row['Z']){
|
||||
// def townObj = isTown(row['X'], row['Y'], row['Z'])
|
||||
// if(townObj){
|
||||
// row['AI'] = "是"
|
||||
// row['AJ'] = row['Z']
|
||||
// }
|
||||
//
|
||||
// def communityObj = findCommunityTown(community_CONN, row['X'], row['Y'], row['Z'])
|
||||
// if(communityObj){
|
||||
// row['AL'] = isGov(communityObj)
|
||||
// if(row['AL'] && row['AL'] =="是"){
|
||||
// row['AM'] = row['Z']
|
||||
// row['AN'] = communityObj.street_code
|
||||
// }
|
||||
// row['AG'] = communityObj.area_name
|
||||
// row['AH'] = communityObj.area_code
|
||||
//
|
||||
// if(row['AI'] && row['AI'] == "是"){
|
||||
// row['AK'] = communityObj.street_code
|
||||
// }
|
||||
// }else{
|
||||
// def towns = findAllTowns(community_CONN, row['X'], row['Y'])
|
||||
// def retObj = findBestSimilar(row['Z'], towns)
|
||||
// if(retObj && retObj.score>=0.5){
|
||||
// row['AQ'] = retObj.name
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
// if(row['F'] && !row['T']){
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("province", row['B'])
|
||||
// def rets = development_zone_CONN.find(q).toList()
|
||||
// def maxScore = 0;
|
||||
// def map = [:]
|
||||
// if(rets){
|
||||
// for(def r : rets){
|
||||
// def score1 = CosineSimilarAlgorithm.getSimilarityRatio(row['B']+row['C']+row['F'], r.name);
|
||||
// if(score1>maxScore){
|
||||
// map.name = r.name
|
||||
// map.score = score1
|
||||
// maxScore = score1
|
||||
// }
|
||||
// }
|
||||
// if(map){
|
||||
// row['W'] = map.name
|
||||
// row['X'] = map.score
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(row['C'] && row['R']){
|
||||
def obj = findFirstData(community_CONN, row['C'], row['D'], row['R'])
|
||||
if(obj){
|
||||
row['S'] = obj.street_code
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
def findBestSimilar(town, towns){
|
||||
def map = [:];
|
||||
def maxScore = 0
|
||||
for(def name : towns){
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(town, name);
|
||||
if(score1>maxScore){
|
||||
map.name = name
|
||||
map.score = score1
|
||||
maxScore = score1
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def isDev(development_zone_CONN, def prov, def city, def region, def dev){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", prov)
|
||||
dev = dev.replace("(","").replace(")","").replace("市","")
|
||||
if(dev.contains(region) && region.toString().endsWith("区")){
|
||||
def newRegion = region.replace("区","")
|
||||
dev = dev.replace(region, newRegion)
|
||||
}
|
||||
Pattern pattern = Pattern.compile('^.*' + dev + '.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("name",new BasicDBObject('$regex', pattern))
|
||||
def rets = development_zone_CONN.find(q).toList()
|
||||
def developmentZone = null
|
||||
if(rets){
|
||||
def devList = []
|
||||
devList.add(dev);
|
||||
try{
|
||||
devList.add(region.replace("县","").replace("区","").replace("市","") + dev);
|
||||
}catch(e){
|
||||
println region +" "+ e.getMessage()
|
||||
}
|
||||
try{
|
||||
devList.add(city.replace("市","") + region.replace("县","").replace("区","").replace("市","") + dev);
|
||||
}catch(e){
|
||||
println city+" "+region+" "+e.getMessage()
|
||||
}
|
||||
try{
|
||||
devList.add(prov.replace("省","") + city.replace("市","")+ region.replace("县","").replace("区","").replace("市","") + dev);
|
||||
}catch(e){
|
||||
println prov+" "+city+" "+region+" "+e.getMessage()
|
||||
}
|
||||
try{
|
||||
devList.add(city.replace("市","") + dev);
|
||||
}catch(e){
|
||||
println e
|
||||
}
|
||||
try{
|
||||
devList.add(prov.replace("省","")+ city.replace("市","") + dev);
|
||||
}catch(e){
|
||||
println e
|
||||
}
|
||||
try{
|
||||
devList.add(prov.replace("省","")+ region.replace("县","").replace("区","").replace("市","")+ dev);
|
||||
}catch(e){
|
||||
println e
|
||||
}
|
||||
devList.add(prov.replace("省","")+ dev);
|
||||
|
||||
for(def zone : rets){
|
||||
String name = zone.name.replace("省","").replace("市","").replace("县","")
|
||||
if(!name.endsWith("区")){
|
||||
name = name.replace("区","");
|
||||
}
|
||||
if(devList.contains(name)){
|
||||
developmentZone = zone;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(developmentZone){
|
||||
return developmentZone.name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def isTown( city, region, town){
|
||||
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
|
||||
// obj = it
|
||||
// },"192.168.0.89")
|
||||
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool":{
|
||||
"must":[
|
||||
{"term": {"city.keyword":"${city}"}},
|
||||
{"term": {"district.keyword":"${region}"}},
|
||||
{"term": {"town.keyword":"${town}"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
def obj = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
obj = hits[0]
|
||||
}
|
||||
}
|
||||
println(response)
|
||||
return obj
|
||||
}
|
||||
|
||||
def findCommunityTown(community_CONN, city, region, town){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", city)
|
||||
q.put("area_name", region)
|
||||
Pattern pattern = Pattern.compile(town + '.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("street_name",new BasicDBObject('$regex', pattern))
|
||||
def rets = community_CONN.find(q).toList()
|
||||
def obj = null
|
||||
if(rets){
|
||||
obj = rets[0]
|
||||
}
|
||||
|
||||
return obj;
|
||||
|
||||
}
|
||||
|
||||
def findAllTowns(community_CONN, city, region){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", city)
|
||||
q.put("area_name", region)
|
||||
def rets = community_CONN.find(q).toList()
|
||||
def towns = []
|
||||
for(def t : rets){
|
||||
if(!towns.contains(t.street_name)){
|
||||
towns << t.street_name
|
||||
}
|
||||
}
|
||||
return towns;
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findFirstData(community_CONN, city, region, town){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", city)
|
||||
q.put("area_name", region)
|
||||
q.put("street_name", town)
|
||||
def obj = community_CONN.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def isGov(obj){
|
||||
String hasGovFlag = null;
|
||||
if(obj){
|
||||
hasGovFlag = obj.has_gov;
|
||||
}
|
||||
if(StringUtils.isNotBlank(hasGovFlag) && hasGovFlag.equals("是")){
|
||||
return "是"
|
||||
}else {
|
||||
return "否"
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
import com.aliyun.tea.*;
|
||||
|
||||
class AnliyunApiUtils {
|
||||
|
||||
/**
|
||||
* 使用AK&SK初始化账号Client
|
||||
* @param accessKeyId
|
||||
* @param accessKeySecret
|
||||
* @return Client
|
||||
* @throws Exception
|
||||
*/
|
||||
public static com.aliyun.iot20180120.Client createClient(String accessKeyId, String accessKeySecret) throws Exception {
|
||||
com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config()
|
||||
// 必填,您的 AccessKey ID
|
||||
.setAccessKeyId(accessKeyId)
|
||||
// 必填,您的 AccessKey Secret
|
||||
.setAccessKeySecret(accessKeySecret);
|
||||
// Endpoint 请参考 https://api.aliyun.com/product/Iot
|
||||
config.endpoint = "iot.cn-beijing.aliyuncs.com";
|
||||
return new com.aliyun.iot20180120.Client(config);
|
||||
}
|
||||
|
||||
public static void main(String[] args_) throws Exception {
|
||||
java.util.List<String> args = java.util.Arrays.asList(args_);
|
||||
// 请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_ID 和 ALIBABA_CLOUD_ACCESS_KEY_SECRET。
|
||||
// 工程代码泄露可能会导致 AccessKey 泄露,并威胁账号下所有资源的安全性。以下代码示例使用环境变量获取 AccessKey 的方式进行调用,仅供参考,建议使用更安全的 STS 方式,更多鉴权访问方式请参见:https://help.aliyun.com/document_detail/378657.html
|
||||
com.aliyun.iot20180120.Client client = createClient(Config.accessKey, Config.accessKeySecret);
|
||||
com.aliyun.iot20180120.models.CreateDeviceGroupRequest createDeviceGroupRequest = new com.aliyun.iot20180120.models.CreateDeviceGroupRequest()
|
||||
.setIotInstanceId("your_value")
|
||||
.setSuperGroupId("your_value")
|
||||
.setGroupDesc("your_value")
|
||||
.setGroupName("your_value");
|
||||
try {
|
||||
// 复制代码运行请自行打印 API 的返回值
|
||||
client.createDeviceGroupWithOptions(createDeviceGroupRequest, new com.aliyun.teautil.models.RuntimeOptions());
|
||||
} catch (TeaException error) {
|
||||
// 错误 message
|
||||
System.out.println(error.getMessage());
|
||||
// 诊断地址
|
||||
System.out.println(error.getData().get("Recommend"));
|
||||
com.aliyun.teautil.Common.assertAsString(error.message);
|
||||
} catch (Exception _error) {
|
||||
TeaException error = new TeaException(_error.getMessage(), _error);
|
||||
// 错误 message
|
||||
System.out.println(error.getMessage());
|
||||
// 诊断地址
|
||||
System.out.println(error.getData().get("Recommend"));
|
||||
com.aliyun.teautil.Common.assertAsString(error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
/* 服务端API签名配置文件
|
||||
*
|
||||
* @author: ali
|
||||
* @version: 0.1 2018-08-08 08:23:54
|
||||
*/
|
||||
public class Config {
|
||||
// AccessKey信息
|
||||
public static String accessKey = "LTAI5t7UKjXaoGxzet2Vh3bB";
|
||||
public static String accessKeySecret = "2avjPxmI7FlIRHvMC5C1UuAnd9iG8z";
|
||||
|
||||
public final static String CHARSET_UTF8 = "utf8";
|
||||
}
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
import com.aliyuncs.CommonRequest
|
||||
import com.aliyuncs.CommonResponse
|
||||
import com.aliyuncs.DefaultAcsClient
|
||||
import com.aliyuncs.IAcsClient
|
||||
import com.aliyuncs.exceptions.ClientException
|
||||
import com.aliyuncs.http.FormatType
|
||||
import com.aliyuncs.http.MethodType
|
||||
import com.aliyuncs.profile.DefaultProfile
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URLDecoder;
|
||||
import java.net.URLEncoder
|
||||
import java.rmi.ServerException;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* 服务端API签名
|
||||
*
|
||||
* @author: ali
|
||||
* @version: 0.1 2018-06-21 20:47:05
|
||||
*/
|
||||
public class SignatureUtils {
|
||||
|
||||
private final static String CHARSET_UTF8 = "utf8";
|
||||
private final static String ALGORITHM = "HmacSHA1";
|
||||
private final static String SEPARATOR = "&";
|
||||
|
||||
public static Map<String, String> splitQueryString(String url)
|
||||
throws URISyntaxException, UnsupportedEncodingException {
|
||||
URI uri = new URI(url);
|
||||
String query = uri.getQuery();
|
||||
final String[] pairs = query.split("&");
|
||||
TreeMap<String, String> queryMap = new TreeMap<String, String>();
|
||||
for (String pair : pairs) {
|
||||
final int idx = pair.indexOf("=");
|
||||
final String key = idx > 0 ? pair.substring(0, idx) : pair;
|
||||
if (!queryMap.containsKey(key)) {
|
||||
queryMap.put(key, URLDecoder.decode(pair.substring(idx + 1), CHARSET_UTF8));
|
||||
}
|
||||
}
|
||||
return queryMap;
|
||||
}
|
||||
|
||||
public static String generate(String method, Map<String, String> parameter, String accessKeySecret)
|
||||
throws Exception {
|
||||
String signString = generateSignString(method, parameter);
|
||||
System.out.println("signString---" + signString);
|
||||
byte[] signBytes = hmacSHA1Signature(accessKeySecret + "&", signString);
|
||||
String signature = newStringByBase64(signBytes);
|
||||
System.out.println("signature----" + signature);
|
||||
if ("POST".equals(method))
|
||||
return signature;
|
||||
return URLEncoder.encode(signature, "UTF-8");
|
||||
}
|
||||
|
||||
public static String generateSignString(String httpMethod, Map<String, String> parameter) throws IOException {
|
||||
TreeMap<String, String> sortParameter = new TreeMap<String, String>();
|
||||
sortParameter.putAll(parameter);
|
||||
String canonicalizedQueryString = UrlUtil.generateQueryString(sortParameter, true);
|
||||
if (null == httpMethod) {
|
||||
throw new RuntimeException("httpMethod can not be empty");
|
||||
}
|
||||
StringBuilder stringToSign = new StringBuilder();
|
||||
stringToSign.append(httpMethod).append(SEPARATOR);
|
||||
stringToSign.append(percentEncode("/")).append(SEPARATOR);
|
||||
stringToSign.append(percentEncode(canonicalizedQueryString));
|
||||
return stringToSign.toString();
|
||||
}
|
||||
|
||||
public static String percentEncode(String value) {
|
||||
try {
|
||||
return value == null ? null
|
||||
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
|
||||
"~");
|
||||
} catch (Exception e) {
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public static byte[] hmacSHA1Signature(String secret, String baseString) throws Exception {
|
||||
if (StringUtils.isEmpty(secret)) {
|
||||
throw new IOException("secret can not be empty");
|
||||
}
|
||||
if (StringUtils.isEmpty(baseString)) {
|
||||
return null;
|
||||
}
|
||||
Mac mac = Mac.getInstance("HmacSHA1");
|
||||
SecretKeySpec keySpec = new SecretKeySpec(secret.getBytes(CHARSET_UTF8), ALGORITHM);
|
||||
mac.init(keySpec);
|
||||
return mac.doFinal(baseString.getBytes(CHARSET_UTF8));
|
||||
}
|
||||
|
||||
public static String newStringByBase64(byte[] bytes) throws UnsupportedEncodingException {
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
return null;
|
||||
}
|
||||
return new String(Base64.encodeBase64(bytes, false), CHARSET_UTF8);
|
||||
}
|
||||
|
||||
|
||||
// public static void main(String[] a){
|
||||
// Map<String, String> map = new HashMap<String, String>();
|
||||
// Calendar calendar = Calendar.getInstance()
|
||||
// def startTime = DateUtil.formatByPattern(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")
|
||||
// // 公共参数
|
||||
// map.put("Format", "JSON");
|
||||
// map.put("Version", "2019-11-18");
|
||||
// map.put("AccessKeyId", Config.accessKey);
|
||||
// map.put("SignatureMethod", "HMAC-SHA1");
|
||||
// map.put("Timestamp", startTime);
|
||||
// map.put("SignatureVersion", "1.0");
|
||||
// map.put("SignatureNonce", System.currentTimeMillis().toString());
|
||||
//// map.put("RegionId", "cn-shanghai");
|
||||
// // 请求参数
|
||||
// map.put("Action", "StreetStd");
|
||||
// map.put("ServiceCode", "addrp");
|
||||
// map.put("AppKey", "Xczmmbxlur2g");
|
||||
// map.put("Text","山东省烟台市莱阳市龙门东路1号0027、0028")
|
||||
// try {
|
||||
// String signature = SignatureUtils.generate("GET", map, Config.accessKeySecret);
|
||||
// if(signature){
|
||||
// map.put("signature", signature);
|
||||
// }
|
||||
// String content = new HttpRequest().get("https://address-purification.cn-hangzhou.aliyuncs.com", map, "utf-8")
|
||||
// JSONObject result = JSONObject.fromObject(content)
|
||||
// System.out.println("最终signature: " + signature);
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// System.out.println();
|
||||
// }
|
||||
|
||||
|
||||
public static def getBaiduAddressStrc(def address) {
|
||||
DefaultProfile defaultProfile = DefaultProfile.getProfile(
|
||||
"cn-hangzhou",
|
||||
Config.accessKey,
|
||||
Config.accessKeySecret);
|
||||
IAcsClient client = new DefaultAcsClient(defaultProfile);
|
||||
// 创建API请求并设置参数
|
||||
CommonRequest request = new CommonRequest();
|
||||
request.setSysDomain("address-purification.cn-hangzhou.aliyuncs.com");
|
||||
request.setSysVersion("2019-11-18");
|
||||
request.setSysProduct("address-purification");
|
||||
request.setSysLocationProduct("addrp");
|
||||
//设置要请求的服务
|
||||
request.setSysAction("StreetStd");
|
||||
//设置请求方式
|
||||
request.setSysMethod(MethodType.POST);
|
||||
request.setHttpContentType(FormatType.FORM);
|
||||
//请求头
|
||||
request.putHeadParameter("x-acs-signature-method", "HMAC-SHA1");
|
||||
request.putHeadParameter("x-acs-signature-nonce", UUID.randomUUID().toString());
|
||||
//设置项目token
|
||||
request.putBodyParameter("AppKey", "Xczmmbxlur2g");
|
||||
//设置请求参数
|
||||
request.putBodyParameter("Text", address);
|
||||
// request.putBodyParameter("DefaultCity", "杭州市");
|
||||
request.putQueryParameter("AcceptLanguage", "zh-CN");
|
||||
|
||||
try {
|
||||
CommonResponse response = client.getCommonResponse(request);
|
||||
def data = response.getData();
|
||||
def result = JSONObject.fromObject(data)
|
||||
if(result && result.Data){
|
||||
def data1 = JSONObject.fromObject(result.Data)
|
||||
def std = data1.street_std
|
||||
if(std){
|
||||
def arrs = std.split("\t")
|
||||
def map = [:]
|
||||
for(def s : arrs){
|
||||
def paramValues = s.split("=")
|
||||
map.put(paramValues[0], paramValues[1])
|
||||
}
|
||||
return map
|
||||
}
|
||||
}
|
||||
|
||||
} catch (ServerException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (ClientException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* URL处理类
|
||||
*
|
||||
* @author: ali
|
||||
* @version: 0.1 2018-06-21 20:40:52
|
||||
*/
|
||||
public class UrlUtil {
|
||||
|
||||
private final static String CHARSET_UTF8 = "utf8";
|
||||
|
||||
public static String urlEncode(String url) {
|
||||
if (!StringUtils.isEmpty(url)) {
|
||||
try {
|
||||
url = URLEncoder.encode(url, "UTF-8");
|
||||
} catch (Exception e) {
|
||||
System.out.println("Url encode error:" + e.getMessage());
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
public static String generateQueryString(Map<String, String> params, boolean isEncodeKV) {
|
||||
StringBuilder canonicalizedQueryString = new StringBuilder();
|
||||
for (Map.Entry<String, String> entry : params.entrySet()) {
|
||||
if (isEncodeKV)
|
||||
canonicalizedQueryString.append(percentEncode(entry.getKey())).append("=")
|
||||
.append(percentEncode(entry.getValue())).append("&");
|
||||
else
|
||||
canonicalizedQueryString.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
|
||||
}
|
||||
if (canonicalizedQueryString.length() > 1) {
|
||||
canonicalizedQueryString.setLength(canonicalizedQueryString.length() - 1);
|
||||
}
|
||||
return canonicalizedQueryString.toString();
|
||||
}
|
||||
|
||||
public static String percentEncode(String value) {
|
||||
try {
|
||||
// 使用URLEncoder.encode编码后,将"+","*","%7E"做替换即满足API规定的编码规范。
|
||||
return value == null ? null
|
||||
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
|
||||
"~");
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return "";
|
||||
}
|
||||
}
|
||||
+17
-13
@@ -7,6 +7,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
@@ -68,21 +69,24 @@ def filePath = "D:\\
|
||||
//})
|
||||
|
||||
|
||||
def objs = COMMUNITY_CONN.find(new BasicDBObject())
|
||||
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
for(def d : objs){
|
||||
if(d.name){
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_pinyin", py1)
|
||||
if(d.location){
|
||||
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
|
||||
udata.put("location", ["lat":lat, "lon":lon])
|
||||
}
|
||||
if(!d.name_pinyin){
|
||||
if(d.name){
|
||||
println d._id
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_pinyin", py1)
|
||||
if(d.location){
|
||||
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
|
||||
udata.put("location", ["lat":lat, "lon":lon])
|
||||
}
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -26,7 +26,7 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
|
||||
|
||||
@@ -58,7 +58,7 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
def getExitsCity(){
|
||||
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||
def cities = SUGGESTION_CONN.distinct("city")
|
||||
def cities = SUGGESTION_CONN.distinct("city", new BasicDBObject("type","poi"))
|
||||
def simpleCities = []
|
||||
for (def c :cities) {
|
||||
def city = RegionData.formatCity(c)
|
||||
@@ -119,7 +119,7 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
try{
|
||||
count = SUGGESTION_CONN.count(q)
|
||||
}catch(e){
|
||||
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
|
||||
count = SUGGESTION_CONN.count(q)
|
||||
}
|
||||
// if(count>0){
|
||||
|
||||
@@ -46,8 +46,11 @@ class ElasticUtils {
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
// connectionProperties.setProperty("user","developer");
|
||||
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
|
||||
connectionProperties.setProperty("user","elastic");
|
||||
connectionProperties.setProperty("password","elasticsearch");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 3
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
Reference in New Issue
Block a user