diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index cd1282a81..785e6d7ae 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -531,6 +531,22 @@ + + com.aliyun + aliyun-java-sdk-address-purification + 1.0.1 + + + com.aliyun + aliyun-java-sdk-core + 4.5.9 + + + com.alibaba + fastjson + 1.2.83 + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index af56de431..dc3ee5d27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -28,7 +28,8 @@ import org.bson.types.ObjectId class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { - public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") +// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") + public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com") public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command') public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017) public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017) @@ -215,7 +216,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { DBObject query = new BasicDBObject() query.put("city", city) query.put("cat", "district") - query.put("visible", 0) + query.put("status", 0) DBObject fields = new BasicDBObject() fields.put("city", 1) @@ -227,11 +228,9 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { int tryTimes = 5 List districts = new ArrayList() for(int i=0; i < tryTimes; i++) { - DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000) + def cur = SEAWEED_COLL.find(query, fields).batchSize(5000) int totalCount = cur.count() - Iterator iter = cur.iterator() - while(iter.hasNext()) { - BasicDBObject dbo = (BasicDBObject)iter.next() + for(def dbo : cur) { districts.add(dbo) } if(districts.size() == totalCount) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 9f88fb0fd..ad59e008c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4007,7 +4007,7 @@ class CommonDatautils { } static def findAmapDetailBySql(city){ - String sql = "select * from amap_poi_detail where cityname='${city}' and status=0" + String sql = "select * from amap_poi where cityname='${city}' and status=0" def rets = [] ElasticUtils.executeSqlQuery(sql,{data-> // if(data.name.contains('华山小区G栋')){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 837c41b27..3abfbaa22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -348,4 +348,24 @@ class FangjiaClientUtils { return jsonObj; } + + public static def villageIdentify(city, address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify?address="+URLEncoder.encode(address,"UTF-8")+"&type=2" + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + println jsonObj + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy new file mode 100644 index 000000000..8a2ee4162 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy @@ -0,0 +1,345 @@ +package com.fangjia.script.gj.village + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.village.ailiyun.SignatureUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +import java.util.regex.Pattern + +def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + +DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + +def filePath = "D:\\数据\\光大\\全量.xlsx" +//ExcelUtils.modify(filePath, { row -> +// if(!row['O']){ +// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A']) +// if(jsonObj && jsonObj.result){ +// row['B'] = jsonObj.result.countyLevelCity +// if(row['B']){ +// if(row['B']=="是"){ +// row['C'] = "否" +// }else if(row['B']=="否"){ +// row['C'] = "是" +// } +// } +// row['D'] = jsonObj.result.district +// row['E'] = jsonObj.result.districtCode +// if(jsonObj.result.administrativeTowns && jsonObj.result.administrativeTowns=="是"){ +// row['F'] = jsonObj.result.administrativeTowns +// row['G'] = jsonObj.result.town +// row['H'] = jsonObj.result.townCode +// } +// +// if(jsonObj.result.countyGov && jsonObj.result.countyGov=="是"){ +// row['I'] = jsonObj.result.countyGov +// row['J'] = jsonObj.result.town +// row['K'] = jsonObj.result.townCode +// } +// +// if(jsonObj.result.devDistrict && jsonObj.result.devDistrict=="是"){ +// row['L'] = jsonObj.result.devDistrict +// row['M'] = jsonObj.result.devDistrictName +// } +// row['N'] = jsonObj.result.addrVerify +// row['O'] = jsonObj.result.province +// row['P'] = jsonObj.result.city +// row['Q'] = jsonObj.result.committee +// row['R'] = jsonObj.result.typeName +// row['S'] = jsonObj.result.splitCommittee +// row['T'] = jsonObj.result.splitStreet +// row['U'] = jsonObj.result.splitStreetNum +// row['V'] = jsonObj.result.poiName +// row['W'] = jsonObj.result.dev +// } +// } + +// def retMap = SignatureUtils.getBaiduAddressStrc(row['A']) +// println row['A'] +" " + retMap.toString() +// if(retMap){ +// row['Y'] = retMap.prov +// row['Z'] = retMap.city +// row['AA'] = retMap.district +// row['AB'] = retMap.town +// row['AC'] = retMap.devzone +// row['AD'] = retMap.road +// row['AE'] = retMap.roadno +// row['AF'] = retMap.poi +// row['AG'] = retMap.subpoi +// } +// }) + + +//ExcelUtils.multiThreadingModify(filePath, { row -> +// +// if(row['AA'] && !row['AO']){ +// if(row['Y']){ +// if(row['Y'].endsWith("区")){ +// row['AE'] = "否" +// row['AF'] = "是" +// }else { +// row['AE'] = "是" +// row['AF'] = "否" +// } +// } +// +// if(row['AA']){ +// def dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], row['AA']) +// if(!dev_name && row['AA'].contains("经济开发区")){ +// dev_name = row['AA'].replace("经济开发区","经济技术开发区") +// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name) +// } +// if(!dev_name && row['AA'].contains("经济技术开发区")){ +// dev_name = row['AA'].replace("经济技术开发区","经济开发区") +// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name) +// } +// if(dev_name){ +// row['AO'] = "是" +// row['AP'] = dev_name +// } +// } +// +// if(row['W'] && row['X'] && row['Y'] && row['Z']){ +// def townObj = isTown(row['X'], row['Y'], row['Z']) +// if(townObj){ +// row['AI'] = "是" +// row['AJ'] = row['Z'] +// } +// +// def communityObj = findCommunityTown(community_CONN, row['X'], row['Y'], row['Z']) +// if(communityObj){ +// row['AL'] = isGov(communityObj) +// if(row['AL'] && row['AL'] =="是"){ +// row['AM'] = row['Z'] +// row['AN'] = communityObj.street_code +// } +// row['AG'] = communityObj.area_name +// row['AH'] = communityObj.area_code +// +// if(row['AI'] && row['AI'] == "是"){ +// row['AK'] = communityObj.street_code +// } +// }else{ +// def towns = findAllTowns(community_CONN, row['X'], row['Y']) +// def retObj = findBestSimilar(row['Z'], towns) +// if(retObj && retObj.score>=0.5){ +// row['AQ'] = retObj.name +// } +// } +// } +// } +// +//}) + + +//ExcelUtils.multiThreadingModify(filePath, { row -> +// if(row['F'] && !row['T']){ +// def q = new BasicDBObject() +// q.put("province", row['B']) +// def rets = development_zone_CONN.find(q).toList() +// def maxScore = 0; +// def map = [:] +// if(rets){ +// for(def r : rets){ +// def score1 = CosineSimilarAlgorithm.getSimilarityRatio(row['B']+row['C']+row['F'], r.name); +// if(score1>maxScore){ +// map.name = r.name +// map.score = score1 +// maxScore = score1 +// } +// } +// if(map){ +// row['W'] = map.name +// row['X'] = map.score +// } +// } +// } +//}) + + +ExcelUtils.multiThreadingModify(filePath, { row -> + if(row['C'] && row['R']){ + def obj = findFirstData(community_CONN, row['C'], row['D'], row['R']) + if(obj){ + row['S'] = obj.street_code + } + } +}) + +def findBestSimilar(town, towns){ + def map = [:]; + def maxScore = 0 + for(def name : towns){ + def score1 = CosineSimilarAlgorithm.getSimilarityRatio(town, name); + if(score1>maxScore){ + map.name = name + map.score = score1 + maxScore = score1 + } + } + return map +} + +def isDev(development_zone_CONN, def prov, def city, def region, def dev){ + def q = new BasicDBObject() + q.put("province", prov) + dev = dev.replace("(","").replace(")","").replace("市","") + if(dev.contains(region) && region.toString().endsWith("区")){ + def newRegion = region.replace("区","") + dev = dev.replace(region, newRegion) + } + Pattern pattern = Pattern.compile('^.*' + dev + '.*$', Pattern.CASE_INSENSITIVE); + q.put("name",new BasicDBObject('$regex', pattern)) + def rets = development_zone_CONN.find(q).toList() + def developmentZone = null + if(rets){ + def devList = [] + devList.add(dev); + try{ + devList.add(region.replace("县","").replace("区","").replace("市","") + dev); + }catch(e){ + println region +" "+ e.getMessage() + } + try{ + devList.add(city.replace("市","") + region.replace("县","").replace("区","").replace("市","") + dev); + }catch(e){ + println city+" "+region+" "+e.getMessage() + } + try{ + devList.add(prov.replace("省","") + city.replace("市","")+ region.replace("县","").replace("区","").replace("市","") + dev); + }catch(e){ + println prov+" "+city+" "+region+" "+e.getMessage() + } + try{ + devList.add(city.replace("市","") + dev); + }catch(e){ + println e + } + try{ + devList.add(prov.replace("省","")+ city.replace("市","") + dev); + }catch(e){ + println e + } + try{ + devList.add(prov.replace("省","")+ region.replace("县","").replace("区","").replace("市","")+ dev); + }catch(e){ + println e + } + devList.add(prov.replace("省","")+ dev); + + for(def zone : rets){ + String name = zone.name.replace("省","").replace("市","").replace("县","") + if(!name.endsWith("区")){ + name = name.replace("区",""); + } + if(devList.contains(name)){ + developmentZone = zone; + break; + } + } + } + if(developmentZone){ + return developmentZone.name; + } + return null; +} + +def isTown( city, region, town){ +// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", { +// obj = it +// },"192.168.0.89") + + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search" + def body = """ + { + "query": { + "bool":{ + "must":[ + {"term": {"city.keyword":"${city}"}}, + {"term": {"district.keyword":"${region}"}}, + {"term": {"town.keyword":"${town}"}} + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = JSONObject.fromObject(res) + def obj = null + if(response && response.hits){ + def hits = response.hits.hits + if(hits){ + obj = hits[0] + } + } + println(response) + return obj +} + +def findCommunityTown(community_CONN, city, region, town){ + + def q = new BasicDBObject() + q.put("city_name", city) + q.put("area_name", region) + Pattern pattern = Pattern.compile(town + '.*$', Pattern.CASE_INSENSITIVE); + q.put("street_name",new BasicDBObject('$regex', pattern)) + def rets = community_CONN.find(q).toList() + def obj = null + if(rets){ + obj = rets[0] + } + + return obj; + +} + +def findAllTowns(community_CONN, city, region){ + + def q = new BasicDBObject() + q.put("city_name", city) + q.put("area_name", region) + def rets = community_CONN.find(q).toList() + def towns = [] + for(def t : rets){ + if(!towns.contains(t.street_name)){ + towns << t.street_name + } + } + return towns; + +} + + +def findFirstData(community_CONN, city, region, town){ + def q = new BasicDBObject() + q.put("city_name", city) + q.put("area_name", region) + q.put("street_name", town) + def obj = community_CONN.findOne(q) + return obj +} + +def isGov(obj){ + String hasGovFlag = null; + if(obj){ + hasGovFlag = obj.has_gov; + } + if(StringUtils.isNotBlank(hasGovFlag) && hasGovFlag.equals("是")){ + return "是" + }else { + return "否" + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/AnliyunApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/AnliyunApiUtils.groovy new file mode 100644 index 000000000..e79251783 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/AnliyunApiUtils.groovy @@ -0,0 +1,53 @@ +package com.fangjia.script.gj.village.ailiyun + +import com.aliyun.tea.*; + +class AnliyunApiUtils { + + /** + * 使用AK&SK初始化账号Client + * @param accessKeyId + * @param accessKeySecret + * @return Client + * @throws Exception + */ + public static com.aliyun.iot20180120.Client createClient(String accessKeyId, String accessKeySecret) throws Exception { + com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config() + // 必填,您的 AccessKey ID + .setAccessKeyId(accessKeyId) + // 必填,您的 AccessKey Secret + .setAccessKeySecret(accessKeySecret); + // Endpoint 请参考 https://api.aliyun.com/product/Iot + config.endpoint = "iot.cn-beijing.aliyuncs.com"; + return new com.aliyun.iot20180120.Client(config); + } + + public static void main(String[] args_) throws Exception { + java.util.List args = java.util.Arrays.asList(args_); + // 请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_ID 和 ALIBABA_CLOUD_ACCESS_KEY_SECRET。 + // 工程代码泄露可能会导致 AccessKey 泄露,并威胁账号下所有资源的安全性。以下代码示例使用环境变量获取 AccessKey 的方式进行调用,仅供参考,建议使用更安全的 STS 方式,更多鉴权访问方式请参见:https://help.aliyun.com/document_detail/378657.html + com.aliyun.iot20180120.Client client = createClient(Config.accessKey, Config.accessKeySecret); + com.aliyun.iot20180120.models.CreateDeviceGroupRequest createDeviceGroupRequest = new com.aliyun.iot20180120.models.CreateDeviceGroupRequest() + .setIotInstanceId("your_value") + .setSuperGroupId("your_value") + .setGroupDesc("your_value") + .setGroupName("your_value"); + try { + // 复制代码运行请自行打印 API 的返回值 + client.createDeviceGroupWithOptions(createDeviceGroupRequest, new com.aliyun.teautil.models.RuntimeOptions()); + } catch (TeaException error) { + // 错误 message + System.out.println(error.getMessage()); + // 诊断地址 + System.out.println(error.getData().get("Recommend")); + com.aliyun.teautil.Common.assertAsString(error.message); + } catch (Exception _error) { + TeaException error = new TeaException(_error.getMessage(), _error); + // 错误 message + System.out.println(error.getMessage()); + // 诊断地址 + System.out.println(error.getData().get("Recommend")); + com.aliyun.teautil.Common.assertAsString(error.message); + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/Config.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/Config.groovy new file mode 100644 index 000000000..e239aab3e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/Config.groovy @@ -0,0 +1,14 @@ +package com.fangjia.script.gj.village.ailiyun + +/* 服务端API签名配置文件 + * + * @author: ali + * @version: 0.1 2018-08-08 08:23:54 + */ +public class Config { + // AccessKey信息 + public static String accessKey = "LTAI5t7UKjXaoGxzet2Vh3bB"; + public static String accessKeySecret = "2avjPxmI7FlIRHvMC5C1UuAnd9iG8z"; + + public final static String CHARSET_UTF8 = "utf8"; +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy new file mode 100644 index 000000000..c0bff3a99 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy @@ -0,0 +1,203 @@ +package com.fangjia.script.gj.village.ailiyun + +import com.aliyuncs.CommonRequest +import com.aliyuncs.CommonResponse +import com.aliyuncs.DefaultAcsClient +import com.aliyuncs.IAcsClient +import com.aliyuncs.exceptions.ClientException +import com.aliyuncs.http.FormatType +import com.aliyuncs.http.MethodType +import com.aliyuncs.profile.DefaultProfile +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.HttpRequest +import net.sf.json.JSONObject + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.URLEncoder +import java.rmi.ServerException; +import java.util.Map; +import java.util.TreeMap; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; + +/** + * 服务端API签名 + * + * @author: ali + * @version: 0.1 2018-06-21 20:47:05 + */ +public class SignatureUtils { + + private final static String CHARSET_UTF8 = "utf8"; + private final static String ALGORITHM = "HmacSHA1"; + private final static String SEPARATOR = "&"; + + public static Map splitQueryString(String url) + throws URISyntaxException, UnsupportedEncodingException { + URI uri = new URI(url); + String query = uri.getQuery(); + final String[] pairs = query.split("&"); + TreeMap queryMap = new TreeMap(); + for (String pair : pairs) { + final int idx = pair.indexOf("="); + final String key = idx > 0 ? pair.substring(0, idx) : pair; + if (!queryMap.containsKey(key)) { + queryMap.put(key, URLDecoder.decode(pair.substring(idx + 1), CHARSET_UTF8)); + } + } + return queryMap; + } + + public static String generate(String method, Map parameter, String accessKeySecret) + throws Exception { + String signString = generateSignString(method, parameter); + System.out.println("signString---" + signString); + byte[] signBytes = hmacSHA1Signature(accessKeySecret + "&", signString); + String signature = newStringByBase64(signBytes); + System.out.println("signature----" + signature); + if ("POST".equals(method)) + return signature; + return URLEncoder.encode(signature, "UTF-8"); + } + + public static String generateSignString(String httpMethod, Map parameter) throws IOException { + TreeMap sortParameter = new TreeMap(); + sortParameter.putAll(parameter); + String canonicalizedQueryString = UrlUtil.generateQueryString(sortParameter, true); + if (null == httpMethod) { + throw new RuntimeException("httpMethod can not be empty"); + } + StringBuilder stringToSign = new StringBuilder(); + stringToSign.append(httpMethod).append(SEPARATOR); + stringToSign.append(percentEncode("/")).append(SEPARATOR); + stringToSign.append(percentEncode(canonicalizedQueryString)); + return stringToSign.toString(); + } + + public static String percentEncode(String value) { + try { + return value == null ? null + : URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E", + "~"); + } catch (Exception e) { + } + return ""; + } + + public static byte[] hmacSHA1Signature(String secret, String baseString) throws Exception { + if (StringUtils.isEmpty(secret)) { + throw new IOException("secret can not be empty"); + } + if (StringUtils.isEmpty(baseString)) { + return null; + } + Mac mac = Mac.getInstance("HmacSHA1"); + SecretKeySpec keySpec = new SecretKeySpec(secret.getBytes(CHARSET_UTF8), ALGORITHM); + mac.init(keySpec); + return mac.doFinal(baseString.getBytes(CHARSET_UTF8)); + } + + public static String newStringByBase64(byte[] bytes) throws UnsupportedEncodingException { + if (bytes == null || bytes.length == 0) { + return null; + } + return new String(Base64.encodeBase64(bytes, false), CHARSET_UTF8); + } + + +// public static void main(String[] a){ +// Map map = new HashMap(); +// Calendar calendar = Calendar.getInstance() +// def startTime = DateUtil.formatByPattern(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ") +// // 公共参数 +// map.put("Format", "JSON"); +// map.put("Version", "2019-11-18"); +// map.put("AccessKeyId", Config.accessKey); +// map.put("SignatureMethod", "HMAC-SHA1"); +// map.put("Timestamp", startTime); +// map.put("SignatureVersion", "1.0"); +// map.put("SignatureNonce", System.currentTimeMillis().toString()); +//// map.put("RegionId", "cn-shanghai"); +// // 请求参数 +// map.put("Action", "StreetStd"); +// map.put("ServiceCode", "addrp"); +// map.put("AppKey", "Xczmmbxlur2g"); +// map.put("Text","山东省烟台市莱阳市龙门东路1号0027、0028") +// try { +// String signature = SignatureUtils.generate("GET", map, Config.accessKeySecret); +// if(signature){ +// map.put("signature", signature); +// } +// String content = new HttpRequest().get("https://address-purification.cn-hangzhou.aliyuncs.com", map, "utf-8") +// JSONObject result = JSONObject.fromObject(content) +// System.out.println("最终signature: " + signature); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// System.out.println(); +// } + + + public static def getBaiduAddressStrc(def address) { + DefaultProfile defaultProfile = DefaultProfile.getProfile( + "cn-hangzhou", + Config.accessKey, + Config.accessKeySecret); + IAcsClient client = new DefaultAcsClient(defaultProfile); + // 创建API请求并设置参数 + CommonRequest request = new CommonRequest(); + request.setSysDomain("address-purification.cn-hangzhou.aliyuncs.com"); + request.setSysVersion("2019-11-18"); + request.setSysProduct("address-purification"); + request.setSysLocationProduct("addrp"); + //设置要请求的服务 + request.setSysAction("StreetStd"); + //设置请求方式 + request.setSysMethod(MethodType.POST); + request.setHttpContentType(FormatType.FORM); + //请求头 + request.putHeadParameter("x-acs-signature-method", "HMAC-SHA1"); + request.putHeadParameter("x-acs-signature-nonce", UUID.randomUUID().toString()); + //设置项目token + request.putBodyParameter("AppKey", "Xczmmbxlur2g"); + //设置请求参数 + request.putBodyParameter("Text", address); +// request.putBodyParameter("DefaultCity", "杭州市"); + request.putQueryParameter("AcceptLanguage", "zh-CN"); + + try { + CommonResponse response = client.getCommonResponse(request); + def data = response.getData(); + def result = JSONObject.fromObject(data) + if(result && result.Data){ + def data1 = JSONObject.fromObject(result.Data) + def std = data1.street_std + if(std){ + def arrs = std.split("\t") + def map = [:] + for(def s : arrs){ + def paramValues = s.split("=") + map.put(paramValues[0], paramValues[1]) + } + return map + } + } + + } catch (ServerException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ClientException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return null; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/UrlUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/UrlUtil.groovy new file mode 100644 index 000000000..0cd2e1fcf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/UrlUtil.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.gj.village.ailiyun + +import java.net.URLEncoder; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +/** + * URL处理类 + * + * @author: ali + * @version: 0.1 2018-06-21 20:40:52 + */ +public class UrlUtil { + + private final static String CHARSET_UTF8 = "utf8"; + + public static String urlEncode(String url) { + if (!StringUtils.isEmpty(url)) { + try { + url = URLEncoder.encode(url, "UTF-8"); + } catch (Exception e) { + System.out.println("Url encode error:" + e.getMessage()); + } + } + return url; + } + + public static String generateQueryString(Map params, boolean isEncodeKV) { + StringBuilder canonicalizedQueryString = new StringBuilder(); + for (Map.Entry entry : params.entrySet()) { + if (isEncodeKV) + canonicalizedQueryString.append(percentEncode(entry.getKey())).append("=") + .append(percentEncode(entry.getValue())).append("&"); + else + canonicalizedQueryString.append(entry.getKey()).append("=").append(entry.getValue()).append("&"); + } + if (canonicalizedQueryString.length() > 1) { + canonicalizedQueryString.setLength(canonicalizedQueryString.length() - 1); + } + return canonicalizedQueryString.toString(); + } + + public static String percentEncode(String value) { + try { + // 使用URLEncoder.encode编码后,将"+","*","%7E"做替换即满足API规定的编码规范。 + return value == null ? null + : URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E", + "~"); + } catch (Exception e) { + + } + return ""; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy index 13dc8a040..785955d49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") @@ -68,21 +69,24 @@ def filePath = "D:\\ //}) -def objs = COMMUNITY_CONN.find(new BasicDBObject()) +def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT); for(def d : objs){ - if(d.name){ - def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ") - def udata = new BasicDBObject() - udata.put("name_pinyin", py1) - if(d.location){ - def lon = DataHandleUtils.getDoubleRound(6, d.location.lon) - def lat = DataHandleUtils.getDoubleRound(6, d.location.lat) - udata.put("location", ["lat":lat, "lon":lon]) - } + if(!d.name_pinyin){ + if(d.name){ + println d._id + def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ") + def udata = new BasicDBObject() + udata.put("name_pinyin", py1) + if(d.location){ + def lon = DataHandleUtils.getDoubleRound(6, d.location.lon) + def lat = DataHandleUtils.getDoubleRound(6, d.location.lat) + udata.put("location", ["lat":lat, "lon":lon]) + } - def q = new BasicDBObject() - q.put("_id", d._id) - COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true) + def q = new BasicDBObject() + q.put("_id", d._id) + COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true) + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 86eac3172..e46d63e3e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -26,7 +26,7 @@ class SuggestionDataSynch extends BaseBatch{ def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) - DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL) + DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL) DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL) @@ -58,7 +58,7 @@ class SuggestionDataSynch extends BaseBatch{ def getExitsCity(){ SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL) - def cities = SUGGESTION_CONN.distinct("city") + def cities = SUGGESTION_CONN.distinct("city", new BasicDBObject("type","poi")) def simpleCities = [] for (def c :cities) { def city = RegionData.formatCity(c) @@ -119,7 +119,7 @@ class SuggestionDataSynch extends BaseBatch{ try{ count = SUGGESTION_CONN.count(q) }catch(e){ - SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL) + SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL) count = SUGGESTION_CONN.count(q) } // if(count>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 6d1ce5312..5e7c817c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -46,8 +46,11 @@ class ElasticUtils { EsDataSource dataSource = new EsDataSource (); dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"); Properties connectionProperties = new Properties(); - connectionProperties.setProperty("user","developer"); - connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); +// connectionProperties.setProperty("user","developer"); +// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); + + connectionProperties.setProperty("user","elastic"); + connectionProperties.setProperty("password","elasticsearch"); dataSource.setProperties(connectionProperties); return dataSource; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index cad37595a..87c14b741 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 5 + def nThread = Runtime.getRuntime().availableProcessors() * 3 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0)