diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml
index 4d279deb8..e04dc02ff 100644
--- a/projects/fangjia-script/pom.xml
+++ b/projects/fangjia-script/pom.xml
@@ -537,6 +537,22 @@
+
+ com.aliyun
+ aliyun-java-sdk-address-purification
+ 1.0.1
+
+
+ com.aliyun
+ aliyun-java-sdk-core
+ 4.5.9
+
+
+ com.alibaba
+ fastjson
+ 1.2.83
+
+
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy
index af56de431..dc3ee5d27 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy
@@ -28,7 +28,8 @@ import org.bson.types.ObjectId
class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
- public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
+// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
+ public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
@@ -215,7 +216,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("cat", "district")
- query.put("visible", 0)
+ query.put("status", 0)
DBObject fields = new BasicDBObject()
fields.put("city", 1)
@@ -227,11 +228,9 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
int tryTimes = 5
List districts = new ArrayList()
for(int i=0; i < tryTimes; i++) {
- DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
+ def cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
int totalCount = cur.count()
- Iterator iter = cur.iterator()
- while(iter.hasNext()) {
- BasicDBObject dbo = (BasicDBObject)iter.next()
+ for(def dbo : cur) {
districts.add(dbo)
}
if(districts.size() == totalCount) {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
index 9f88fb0fd..ad59e008c 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
@@ -4007,7 +4007,7 @@ class CommonDatautils {
}
static def findAmapDetailBySql(city){
- String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
+ String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
// if(data.name.contains('华山小区G栋')){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy
index 837c41b27..3abfbaa22 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy
@@ -348,4 +348,24 @@ class FangjiaClientUtils {
return jsonObj;
}
+
+ public static def villageIdentify(city, address){
+ def jsonObj = null
+ def content = null
+ try {
+ address = address.replace("{","")
+ def url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify?address="+URLEncoder.encode(address,"UTF-8")+"&type=2"
+ if(city){
+ url = url +"&city="+URLEncoder.encode(city,"UTF-8")
+ }
+ content = new HttpRequest().newGet(url, null, "utf-8")
+ jsonObj = JSONObject.fromObject(content)
+ println jsonObj
+ } catch (Exception e) {
+ println address +" "+content
+ e.printStackTrace();
+ }
+ return jsonObj;
+ }
+
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy
new file mode 100644
index 000000000..8a2ee4162
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy
@@ -0,0 +1,345 @@
+package com.fangjia.script.gj.village
+
+import com.fangjia.core.util.HttpRequest
+import com.fangjia.script.gj.district.FangjiaClientUtils
+import com.fangjia.script.gj.village.ailiyun.SignatureUtils
+import com.fangjia.script.util.CosineSimilarAlgorithm
+import com.fangjia.script.util.ElasticUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import net.sf.json.JSONObject
+import org.apache.commons.lang.StringUtils
+
+import java.util.regex.Pattern
+
+def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
+DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
+
+DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
+
+
+def filePath = "D:\\数据\\光大\\全量.xlsx"
+//ExcelUtils.modify(filePath, { row ->
+// if(!row['O']){
+// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A'])
+// if(jsonObj && jsonObj.result){
+// row['B'] = jsonObj.result.countyLevelCity
+// if(row['B']){
+// if(row['B']=="是"){
+// row['C'] = "否"
+// }else if(row['B']=="否"){
+// row['C'] = "是"
+// }
+// }
+// row['D'] = jsonObj.result.district
+// row['E'] = jsonObj.result.districtCode
+// if(jsonObj.result.administrativeTowns && jsonObj.result.administrativeTowns=="是"){
+// row['F'] = jsonObj.result.administrativeTowns
+// row['G'] = jsonObj.result.town
+// row['H'] = jsonObj.result.townCode
+// }
+//
+// if(jsonObj.result.countyGov && jsonObj.result.countyGov=="是"){
+// row['I'] = jsonObj.result.countyGov
+// row['J'] = jsonObj.result.town
+// row['K'] = jsonObj.result.townCode
+// }
+//
+// if(jsonObj.result.devDistrict && jsonObj.result.devDistrict=="是"){
+// row['L'] = jsonObj.result.devDistrict
+// row['M'] = jsonObj.result.devDistrictName
+// }
+// row['N'] = jsonObj.result.addrVerify
+// row['O'] = jsonObj.result.province
+// row['P'] = jsonObj.result.city
+// row['Q'] = jsonObj.result.committee
+// row['R'] = jsonObj.result.typeName
+// row['S'] = jsonObj.result.splitCommittee
+// row['T'] = jsonObj.result.splitStreet
+// row['U'] = jsonObj.result.splitStreetNum
+// row['V'] = jsonObj.result.poiName
+// row['W'] = jsonObj.result.dev
+// }
+// }
+
+// def retMap = SignatureUtils.getBaiduAddressStrc(row['A'])
+// println row['A'] +" " + retMap.toString()
+// if(retMap){
+// row['Y'] = retMap.prov
+// row['Z'] = retMap.city
+// row['AA'] = retMap.district
+// row['AB'] = retMap.town
+// row['AC'] = retMap.devzone
+// row['AD'] = retMap.road
+// row['AE'] = retMap.roadno
+// row['AF'] = retMap.poi
+// row['AG'] = retMap.subpoi
+// }
+// })
+
+
+//ExcelUtils.multiThreadingModify(filePath, { row ->
+//
+// if(row['AA'] && !row['AO']){
+// if(row['Y']){
+// if(row['Y'].endsWith("区")){
+// row['AE'] = "否"
+// row['AF'] = "是"
+// }else {
+// row['AE'] = "是"
+// row['AF'] = "否"
+// }
+// }
+//
+// if(row['AA']){
+// def dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], row['AA'])
+// if(!dev_name && row['AA'].contains("经济开发区")){
+// dev_name = row['AA'].replace("经济开发区","经济技术开发区")
+// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
+// }
+// if(!dev_name && row['AA'].contains("经济技术开发区")){
+// dev_name = row['AA'].replace("经济技术开发区","经济开发区")
+// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
+// }
+// if(dev_name){
+// row['AO'] = "是"
+// row['AP'] = dev_name
+// }
+// }
+//
+// if(row['W'] && row['X'] && row['Y'] && row['Z']){
+// def townObj = isTown(row['X'], row['Y'], row['Z'])
+// if(townObj){
+// row['AI'] = "是"
+// row['AJ'] = row['Z']
+// }
+//
+// def communityObj = findCommunityTown(community_CONN, row['X'], row['Y'], row['Z'])
+// if(communityObj){
+// row['AL'] = isGov(communityObj)
+// if(row['AL'] && row['AL'] =="是"){
+// row['AM'] = row['Z']
+// row['AN'] = communityObj.street_code
+// }
+// row['AG'] = communityObj.area_name
+// row['AH'] = communityObj.area_code
+//
+// if(row['AI'] && row['AI'] == "是"){
+// row['AK'] = communityObj.street_code
+// }
+// }else{
+// def towns = findAllTowns(community_CONN, row['X'], row['Y'])
+// def retObj = findBestSimilar(row['Z'], towns)
+// if(retObj && retObj.score>=0.5){
+// row['AQ'] = retObj.name
+// }
+// }
+// }
+// }
+//
+//})
+
+
+//ExcelUtils.multiThreadingModify(filePath, { row ->
+// if(row['F'] && !row['T']){
+// def q = new BasicDBObject()
+// q.put("province", row['B'])
+// def rets = development_zone_CONN.find(q).toList()
+// def maxScore = 0;
+// def map = [:]
+// if(rets){
+// for(def r : rets){
+// def score1 = CosineSimilarAlgorithm.getSimilarityRatio(row['B']+row['C']+row['F'], r.name);
+// if(score1>maxScore){
+// map.name = r.name
+// map.score = score1
+// maxScore = score1
+// }
+// }
+// if(map){
+// row['W'] = map.name
+// row['X'] = map.score
+// }
+// }
+// }
+//})
+
+
+ExcelUtils.multiThreadingModify(filePath, { row ->
+ if(row['C'] && row['R']){
+ def obj = findFirstData(community_CONN, row['C'], row['D'], row['R'])
+ if(obj){
+ row['S'] = obj.street_code
+ }
+ }
+})
+
+def findBestSimilar(town, towns){
+ def map = [:];
+ def maxScore = 0
+ for(def name : towns){
+ def score1 = CosineSimilarAlgorithm.getSimilarityRatio(town, name);
+ if(score1>maxScore){
+ map.name = name
+ map.score = score1
+ maxScore = score1
+ }
+ }
+ return map
+}
+
+def isDev(development_zone_CONN, def prov, def city, def region, def dev){
+ def q = new BasicDBObject()
+ q.put("province", prov)
+ dev = dev.replace("(","").replace(")","").replace("市","")
+ if(dev.contains(region) && region.toString().endsWith("区")){
+ def newRegion = region.replace("区","")
+ dev = dev.replace(region, newRegion)
+ }
+ Pattern pattern = Pattern.compile('^.*' + dev + '.*$', Pattern.CASE_INSENSITIVE);
+ q.put("name",new BasicDBObject('$regex', pattern))
+ def rets = development_zone_CONN.find(q).toList()
+ def developmentZone = null
+ if(rets){
+ def devList = []
+ devList.add(dev);
+ try{
+ devList.add(region.replace("县","").replace("区","").replace("市","") + dev);
+ }catch(e){
+ println region +" "+ e.getMessage()
+ }
+ try{
+ devList.add(city.replace("市","") + region.replace("县","").replace("区","").replace("市","") + dev);
+ }catch(e){
+ println city+" "+region+" "+e.getMessage()
+ }
+ try{
+ devList.add(prov.replace("省","") + city.replace("市","")+ region.replace("县","").replace("区","").replace("市","") + dev);
+ }catch(e){
+ println prov+" "+city+" "+region+" "+e.getMessage()
+ }
+ try{
+ devList.add(city.replace("市","") + dev);
+ }catch(e){
+ println e
+ }
+ try{
+ devList.add(prov.replace("省","")+ city.replace("市","") + dev);
+ }catch(e){
+ println e
+ }
+ try{
+ devList.add(prov.replace("省","")+ region.replace("县","").replace("区","").replace("市","")+ dev);
+ }catch(e){
+ println e
+ }
+ devList.add(prov.replace("省","")+ dev);
+
+ for(def zone : rets){
+ String name = zone.name.replace("省","").replace("市","").replace("县","")
+ if(!name.endsWith("区")){
+ name = name.replace("区","");
+ }
+ if(devList.contains(name)){
+ developmentZone = zone;
+ break;
+ }
+ }
+ }
+ if(developmentZone){
+ return developmentZone.name;
+ }
+ return null;
+}
+
+def isTown( city, region, town){
+// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
+// obj = it
+// },"192.168.0.89")
+
+ def header = [:]
+ header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
+ String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
+ def body = """
+ {
+ "query": {
+ "bool":{
+ "must":[
+ {"term": {"city.keyword":"${city}"}},
+ {"term": {"district.keyword":"${region}"}},
+ {"term": {"town.keyword":"${town}"}}
+ ]
+ }
+ }
+ }
+ """
+ HttpRequest request = new HttpRequest()
+ String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
+ def response = JSONObject.fromObject(res)
+ def obj = null
+ if(response && response.hits){
+ def hits = response.hits.hits
+ if(hits){
+ obj = hits[0]
+ }
+ }
+ println(response)
+ return obj
+}
+
+def findCommunityTown(community_CONN, city, region, town){
+
+ def q = new BasicDBObject()
+ q.put("city_name", city)
+ q.put("area_name", region)
+ Pattern pattern = Pattern.compile(town + '.*$', Pattern.CASE_INSENSITIVE);
+ q.put("street_name",new BasicDBObject('$regex', pattern))
+ def rets = community_CONN.find(q).toList()
+ def obj = null
+ if(rets){
+ obj = rets[0]
+ }
+
+ return obj;
+
+}
+
+def findAllTowns(community_CONN, city, region){
+
+ def q = new BasicDBObject()
+ q.put("city_name", city)
+ q.put("area_name", region)
+ def rets = community_CONN.find(q).toList()
+ def towns = []
+ for(def t : rets){
+ if(!towns.contains(t.street_name)){
+ towns << t.street_name
+ }
+ }
+ return towns;
+
+}
+
+
+def findFirstData(community_CONN, city, region, town){
+ def q = new BasicDBObject()
+ q.put("city_name", city)
+ q.put("area_name", region)
+ q.put("street_name", town)
+ def obj = community_CONN.findOne(q)
+ return obj
+}
+
+def isGov(obj){
+ String hasGovFlag = null;
+ if(obj){
+ hasGovFlag = obj.has_gov;
+ }
+ if(StringUtils.isNotBlank(hasGovFlag) && hasGovFlag.equals("是")){
+ return "是"
+ }else {
+ return "否"
+ }
+}
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/AnliyunApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/AnliyunApiUtils.groovy
new file mode 100644
index 000000000..e79251783
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/AnliyunApiUtils.groovy
@@ -0,0 +1,53 @@
+package com.fangjia.script.gj.village.ailiyun
+
+import com.aliyun.tea.*;
+
+class AnliyunApiUtils {
+
+ /**
+ * 使用AK&SK初始化账号Client
+ * @param accessKeyId
+ * @param accessKeySecret
+ * @return Client
+ * @throws Exception
+ */
+ public static com.aliyun.iot20180120.Client createClient(String accessKeyId, String accessKeySecret) throws Exception {
+ com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config()
+ // 必填,您的 AccessKey ID
+ .setAccessKeyId(accessKeyId)
+ // 必填,您的 AccessKey Secret
+ .setAccessKeySecret(accessKeySecret);
+ // Endpoint 请参考 https://api.aliyun.com/product/Iot
+ config.endpoint = "iot.cn-beijing.aliyuncs.com";
+ return new com.aliyun.iot20180120.Client(config);
+ }
+
+ public static void main(String[] args_) throws Exception {
+ java.util.List args = java.util.Arrays.asList(args_);
+ // 请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_ID 和 ALIBABA_CLOUD_ACCESS_KEY_SECRET。
+ // 工程代码泄露可能会导致 AccessKey 泄露,并威胁账号下所有资源的安全性。以下代码示例使用环境变量获取 AccessKey 的方式进行调用,仅供参考,建议使用更安全的 STS 方式,更多鉴权访问方式请参见:https://help.aliyun.com/document_detail/378657.html
+ com.aliyun.iot20180120.Client client = createClient(Config.accessKey, Config.accessKeySecret);
+ com.aliyun.iot20180120.models.CreateDeviceGroupRequest createDeviceGroupRequest = new com.aliyun.iot20180120.models.CreateDeviceGroupRequest()
+ .setIotInstanceId("your_value")
+ .setSuperGroupId("your_value")
+ .setGroupDesc("your_value")
+ .setGroupName("your_value");
+ try {
+ // 复制代码运行请自行打印 API 的返回值
+ client.createDeviceGroupWithOptions(createDeviceGroupRequest, new com.aliyun.teautil.models.RuntimeOptions());
+ } catch (TeaException error) {
+ // 错误 message
+ System.out.println(error.getMessage());
+ // 诊断地址
+ System.out.println(error.getData().get("Recommend"));
+ com.aliyun.teautil.Common.assertAsString(error.message);
+ } catch (Exception _error) {
+ TeaException error = new TeaException(_error.getMessage(), _error);
+ // 错误 message
+ System.out.println(error.getMessage());
+ // 诊断地址
+ System.out.println(error.getData().get("Recommend"));
+ com.aliyun.teautil.Common.assertAsString(error.message);
+ }
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/Config.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/Config.groovy
new file mode 100644
index 000000000..e239aab3e
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/Config.groovy
@@ -0,0 +1,14 @@
+package com.fangjia.script.gj.village.ailiyun
+
+/* 服务端API签名配置文件
+ *
+ * @author: ali
+ * @version: 0.1 2018-08-08 08:23:54
+ */
+public class Config {
+ // AccessKey信息
+ public static String accessKey = "LTAI5t7UKjXaoGxzet2Vh3bB";
+ public static String accessKeySecret = "2avjPxmI7FlIRHvMC5C1UuAnd9iG8z";
+
+ public final static String CHARSET_UTF8 = "utf8";
+}
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy
new file mode 100644
index 000000000..c0bff3a99
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy
@@ -0,0 +1,203 @@
+package com.fangjia.script.gj.village.ailiyun
+
+import com.aliyuncs.CommonRequest
+import com.aliyuncs.CommonResponse
+import com.aliyuncs.DefaultAcsClient
+import com.aliyuncs.IAcsClient
+import com.aliyuncs.exceptions.ClientException
+import com.aliyuncs.http.FormatType
+import com.aliyuncs.http.MethodType
+import com.aliyuncs.profile.DefaultProfile
+import com.fangjia.core.util.DateUtil
+import com.fangjia.core.util.HttpRequest
+import net.sf.json.JSONObject
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLDecoder;
+import java.net.URLEncoder
+import java.rmi.ServerException;
+import java.util.Map;
+import java.util.TreeMap;
+
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * 服务端API签名
+ *
+ * @author: ali
+ * @version: 0.1 2018-06-21 20:47:05
+ */
+public class SignatureUtils {
+
+ private final static String CHARSET_UTF8 = "utf8";
+ private final static String ALGORITHM = "HmacSHA1";
+ private final static String SEPARATOR = "&";
+
+ public static Map splitQueryString(String url)
+ throws URISyntaxException, UnsupportedEncodingException {
+ URI uri = new URI(url);
+ String query = uri.getQuery();
+ final String[] pairs = query.split("&");
+ TreeMap queryMap = new TreeMap();
+ for (String pair : pairs) {
+ final int idx = pair.indexOf("=");
+ final String key = idx > 0 ? pair.substring(0, idx) : pair;
+ if (!queryMap.containsKey(key)) {
+ queryMap.put(key, URLDecoder.decode(pair.substring(idx + 1), CHARSET_UTF8));
+ }
+ }
+ return queryMap;
+ }
+
+ public static String generate(String method, Map parameter, String accessKeySecret)
+ throws Exception {
+ String signString = generateSignString(method, parameter);
+ System.out.println("signString---" + signString);
+ byte[] signBytes = hmacSHA1Signature(accessKeySecret + "&", signString);
+ String signature = newStringByBase64(signBytes);
+ System.out.println("signature----" + signature);
+ if ("POST".equals(method))
+ return signature;
+ return URLEncoder.encode(signature, "UTF-8");
+ }
+
+ public static String generateSignString(String httpMethod, Map parameter) throws IOException {
+ TreeMap sortParameter = new TreeMap();
+ sortParameter.putAll(parameter);
+ String canonicalizedQueryString = UrlUtil.generateQueryString(sortParameter, true);
+ if (null == httpMethod) {
+ throw new RuntimeException("httpMethod can not be empty");
+ }
+ StringBuilder stringToSign = new StringBuilder();
+ stringToSign.append(httpMethod).append(SEPARATOR);
+ stringToSign.append(percentEncode("/")).append(SEPARATOR);
+ stringToSign.append(percentEncode(canonicalizedQueryString));
+ return stringToSign.toString();
+ }
+
+ public static String percentEncode(String value) {
+ try {
+ return value == null ? null
+ : URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
+ "~");
+ } catch (Exception e) {
+ }
+ return "";
+ }
+
+ public static byte[] hmacSHA1Signature(String secret, String baseString) throws Exception {
+ if (StringUtils.isEmpty(secret)) {
+ throw new IOException("secret can not be empty");
+ }
+ if (StringUtils.isEmpty(baseString)) {
+ return null;
+ }
+ Mac mac = Mac.getInstance("HmacSHA1");
+ SecretKeySpec keySpec = new SecretKeySpec(secret.getBytes(CHARSET_UTF8), ALGORITHM);
+ mac.init(keySpec);
+ return mac.doFinal(baseString.getBytes(CHARSET_UTF8));
+ }
+
+ public static String newStringByBase64(byte[] bytes) throws UnsupportedEncodingException {
+ if (bytes == null || bytes.length == 0) {
+ return null;
+ }
+ return new String(Base64.encodeBase64(bytes, false), CHARSET_UTF8);
+ }
+
+
+// public static void main(String[] a){
+// Map map = new HashMap();
+// Calendar calendar = Calendar.getInstance()
+// def startTime = DateUtil.formatByPattern(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")
+// // 公共参数
+// map.put("Format", "JSON");
+// map.put("Version", "2019-11-18");
+// map.put("AccessKeyId", Config.accessKey);
+// map.put("SignatureMethod", "HMAC-SHA1");
+// map.put("Timestamp", startTime);
+// map.put("SignatureVersion", "1.0");
+// map.put("SignatureNonce", System.currentTimeMillis().toString());
+//// map.put("RegionId", "cn-shanghai");
+// // 请求参数
+// map.put("Action", "StreetStd");
+// map.put("ServiceCode", "addrp");
+// map.put("AppKey", "Xczmmbxlur2g");
+// map.put("Text","山东省烟台市莱阳市龙门东路1号0027、0028")
+// try {
+// String signature = SignatureUtils.generate("GET", map, Config.accessKeySecret);
+// if(signature){
+// map.put("signature", signature);
+// }
+// String content = new HttpRequest().get("https://address-purification.cn-hangzhou.aliyuncs.com", map, "utf-8")
+// JSONObject result = JSONObject.fromObject(content)
+// System.out.println("最终signature: " + signature);
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// System.out.println();
+// }
+
+
+ public static def getBaiduAddressStrc(def address) {
+ DefaultProfile defaultProfile = DefaultProfile.getProfile(
+ "cn-hangzhou",
+ Config.accessKey,
+ Config.accessKeySecret);
+ IAcsClient client = new DefaultAcsClient(defaultProfile);
+ // 创建API请求并设置参数
+ CommonRequest request = new CommonRequest();
+ request.setSysDomain("address-purification.cn-hangzhou.aliyuncs.com");
+ request.setSysVersion("2019-11-18");
+ request.setSysProduct("address-purification");
+ request.setSysLocationProduct("addrp");
+ //设置要请求的服务
+ request.setSysAction("StreetStd");
+ //设置请求方式
+ request.setSysMethod(MethodType.POST);
+ request.setHttpContentType(FormatType.FORM);
+ //请求头
+ request.putHeadParameter("x-acs-signature-method", "HMAC-SHA1");
+ request.putHeadParameter("x-acs-signature-nonce", UUID.randomUUID().toString());
+ //设置项目token
+ request.putBodyParameter("AppKey", "Xczmmbxlur2g");
+ //设置请求参数
+ request.putBodyParameter("Text", address);
+// request.putBodyParameter("DefaultCity", "杭州市");
+ request.putQueryParameter("AcceptLanguage", "zh-CN");
+
+ try {
+ CommonResponse response = client.getCommonResponse(request);
+ def data = response.getData();
+ def result = JSONObject.fromObject(data)
+ if(result && result.Data){
+ def data1 = JSONObject.fromObject(result.Data)
+ def std = data1.street_std
+ if(std){
+ def arrs = std.split("\t")
+ def map = [:]
+ for(def s : arrs){
+ def paramValues = s.split("=")
+ map.put(paramValues[0], paramValues[1])
+ }
+ return map
+ }
+ }
+
+ } catch (ServerException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (ClientException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return null;
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/UrlUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/UrlUtil.groovy
new file mode 100644
index 000000000..0cd2e1fcf
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/UrlUtil.groovy
@@ -0,0 +1,55 @@
+package com.fangjia.script.gj.village.ailiyun
+
+import java.net.URLEncoder;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * URL处理类
+ *
+ * @author: ali
+ * @version: 0.1 2018-06-21 20:40:52
+ */
+public class UrlUtil {
+
+ private final static String CHARSET_UTF8 = "utf8";
+
+ public static String urlEncode(String url) {
+ if (!StringUtils.isEmpty(url)) {
+ try {
+ url = URLEncoder.encode(url, "UTF-8");
+ } catch (Exception e) {
+ System.out.println("Url encode error:" + e.getMessage());
+ }
+ }
+ return url;
+ }
+
+ public static String generateQueryString(Map params, boolean isEncodeKV) {
+ StringBuilder canonicalizedQueryString = new StringBuilder();
+ for (Map.Entry entry : params.entrySet()) {
+ if (isEncodeKV)
+ canonicalizedQueryString.append(percentEncode(entry.getKey())).append("=")
+ .append(percentEncode(entry.getValue())).append("&");
+ else
+ canonicalizedQueryString.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
+ }
+ if (canonicalizedQueryString.length() > 1) {
+ canonicalizedQueryString.setLength(canonicalizedQueryString.length() - 1);
+ }
+ return canonicalizedQueryString.toString();
+ }
+
+ public static String percentEncode(String value) {
+ try {
+ // 使用URLEncoder.encode编码后,将"+","*","%7E"做替换即满足API规定的编码规范。
+ return value == null ? null
+ : URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
+ "~");
+ } catch (Exception e) {
+
+ }
+ return "";
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy
index 13dc8a040..785955d49 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy
@@ -7,6 +7,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
+import com.mongodb.Bytes
import com.mongodb.DBCollection
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
@@ -68,21 +69,24 @@ def filePath = "D:\\
//})
-def objs = COMMUNITY_CONN.find(new BasicDBObject())
+def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
for(def d : objs){
- if(d.name){
- def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
- def udata = new BasicDBObject()
- udata.put("name_pinyin", py1)
- if(d.location){
- def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
- def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
- udata.put("location", ["lat":lat, "lon":lon])
- }
+ if(!d.name_pinyin){
+ if(d.name){
+ println d._id
+ def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
+ def udata = new BasicDBObject()
+ udata.put("name_pinyin", py1)
+ if(d.location){
+ def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
+ def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
+ udata.put("location", ["lat":lat, "lon":lon])
+ }
- def q = new BasicDBObject()
- q.put("_id", d._id)
- COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
+ def q = new BasicDBObject()
+ q.put("_id", d._id)
+ COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
+ }
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy
index 86eac3172..8aa6511ac 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy
@@ -26,7 +26,7 @@ class SuggestionDataSynch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
- DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
+ DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
@@ -57,8 +57,8 @@ class SuggestionDataSynch extends BaseBatch{
}
def getExitsCity(){
- SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
- def cities = SUGGESTION_CONN.distinct("city")
+ SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
+ def cities = SUGGESTION_CONN.distinct("city", new BasicDBObject("type","poi"))
def simpleCities = []
for (def c :cities) {
def city = RegionData.formatCity(c)
@@ -119,7 +119,7 @@ class SuggestionDataSynch extends BaseBatch{
try{
count = SUGGESTION_CONN.count(q)
}catch(e){
- SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
+ SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
count = SUGGESTION_CONN.count(q)
}
// if(count>0){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy
index 6d1ce5312..f14bedadd 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy
@@ -46,8 +46,11 @@ class ElasticUtils {
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
- connectionProperties.setProperty("user","developer");
- connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
+// connectionProperties.setProperty("user","developer");
+// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
+
+ connectionProperties.setProperty("user","elastic");
+ connectionProperties.setProperty("password","goojia123456");//elasticsearch
dataSource.setProperties(connectionProperties);
return dataSource;
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy
index cad37595a..87c14b741 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
- def nThread = Runtime.getRuntime().availableProcessors() * 5
+ def nThread = Runtime.getRuntime().availableProcessors() * 3
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)