Merge branch 'script_gj' into 'master'

Script gj

See merge request !807
This commit is contained in:
2024-02-18 15:45:46 +08:00
13 changed files with 739 additions and 27 deletions
+16
View File
@@ -537,6 +537,22 @@
<!-- <scope>compile</scope>-->
<!-- </dependency>-->
<dependency>
<groupId>com.aliyun</groupId>
<artifactId>aliyun-java-sdk-address-purification</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>com.aliyun</groupId>
<artifactId>aliyun-java-sdk-core</artifactId>
<version>4.5.9</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.83</version>
</dependency>
</dependencies>
</project>
@@ -28,7 +28,8 @@ import org.bson.types.ObjectId
class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
@@ -215,7 +216,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("cat", "district")
query.put("visible", 0)
query.put("status", 0)
DBObject fields = new BasicDBObject()
fields.put("city", 1)
@@ -227,11 +228,9 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
int tryTimes = 5
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
for(int i=0; i < tryTimes; i++) {
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
def cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
int totalCount = cur.count()
Iterator<DBObject> iter = cur.iterator()
while(iter.hasNext()) {
BasicDBObject dbo = (BasicDBObject)iter.next()
for(def dbo : cur) {
districts.add(dbo)
}
if(districts.size() == totalCount) {
@@ -4007,7 +4007,7 @@ class CommonDatautils {
}
static def findAmapDetailBySql(city){
String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
// if(data.name.contains('华山小区G栋')){
@@ -348,4 +348,24 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def villageIdentify(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify?address="+URLEncoder.encode(address,"UTF-8")+"&type=2"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -0,0 +1,345 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def filePath = "D:\\数据\\光大\\全量.xlsx"
//ExcelUtils.modify(filePath, { row ->
// if(!row['O']){
// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A'])
// if(jsonObj && jsonObj.result){
// row['B'] = jsonObj.result.countyLevelCity
// if(row['B']){
// if(row['B']=="是"){
// row['C'] = "否"
// }else if(row['B']=="否"){
// row['C'] = "是"
// }
// }
// row['D'] = jsonObj.result.district
// row['E'] = jsonObj.result.districtCode
// if(jsonObj.result.administrativeTowns && jsonObj.result.administrativeTowns=="是"){
// row['F'] = jsonObj.result.administrativeTowns
// row['G'] = jsonObj.result.town
// row['H'] = jsonObj.result.townCode
// }
//
// if(jsonObj.result.countyGov && jsonObj.result.countyGov=="是"){
// row['I'] = jsonObj.result.countyGov
// row['J'] = jsonObj.result.town
// row['K'] = jsonObj.result.townCode
// }
//
// if(jsonObj.result.devDistrict && jsonObj.result.devDistrict=="是"){
// row['L'] = jsonObj.result.devDistrict
// row['M'] = jsonObj.result.devDistrictName
// }
// row['N'] = jsonObj.result.addrVerify
// row['O'] = jsonObj.result.province
// row['P'] = jsonObj.result.city
// row['Q'] = jsonObj.result.committee
// row['R'] = jsonObj.result.typeName
// row['S'] = jsonObj.result.splitCommittee
// row['T'] = jsonObj.result.splitStreet
// row['U'] = jsonObj.result.splitStreetNum
// row['V'] = jsonObj.result.poiName
// row['W'] = jsonObj.result.dev
// }
// }
// def retMap = SignatureUtils.getBaiduAddressStrc(row['A'])
// println row['A'] +" " + retMap.toString()
// if(retMap){
// row['Y'] = retMap.prov
// row['Z'] = retMap.city
// row['AA'] = retMap.district
// row['AB'] = retMap.town
// row['AC'] = retMap.devzone
// row['AD'] = retMap.road
// row['AE'] = retMap.roadno
// row['AF'] = retMap.poi
// row['AG'] = retMap.subpoi
// }
// })
//ExcelUtils.multiThreadingModify(filePath, { row ->
//
// if(row['AA'] && !row['AO']){
// if(row['Y']){
// if(row['Y'].endsWith("区")){
// row['AE'] = "否"
// row['AF'] = "是"
// }else {
// row['AE'] = "是"
// row['AF'] = "否"
// }
// }
//
// if(row['AA']){
// def dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], row['AA'])
// if(!dev_name && row['AA'].contains("经济开发区")){
// dev_name = row['AA'].replace("经济开发区","经济技术开发区")
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
// }
// if(!dev_name && row['AA'].contains("经济技术开发区")){
// dev_name = row['AA'].replace("经济技术开发区","经济开发区")
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
// }
// if(dev_name){
// row['AO'] = "是"
// row['AP'] = dev_name
// }
// }
//
// if(row['W'] && row['X'] && row['Y'] && row['Z']){
// def townObj = isTown(row['X'], row['Y'], row['Z'])
// if(townObj){
// row['AI'] = "是"
// row['AJ'] = row['Z']
// }
//
// def communityObj = findCommunityTown(community_CONN, row['X'], row['Y'], row['Z'])
// if(communityObj){
// row['AL'] = isGov(communityObj)
// if(row['AL'] && row['AL'] =="是"){
// row['AM'] = row['Z']
// row['AN'] = communityObj.street_code
// }
// row['AG'] = communityObj.area_name
// row['AH'] = communityObj.area_code
//
// if(row['AI'] && row['AI'] == "是"){
// row['AK'] = communityObj.street_code
// }
// }else{
// def towns = findAllTowns(community_CONN, row['X'], row['Y'])
// def retObj = findBestSimilar(row['Z'], towns)
// if(retObj && retObj.score>=0.5){
// row['AQ'] = retObj.name
// }
// }
// }
// }
//
//})
//ExcelUtils.multiThreadingModify(filePath, { row ->
// if(row['F'] && !row['T']){
// def q = new BasicDBObject()
// q.put("province", row['B'])
// def rets = development_zone_CONN.find(q).toList()
// def maxScore = 0;
// def map = [:]
// if(rets){
// for(def r : rets){
// def score1 = CosineSimilarAlgorithm.getSimilarityRatio(row['B']+row['C']+row['F'], r.name);
// if(score1>maxScore){
// map.name = r.name
// map.score = score1
// maxScore = score1
// }
// }
// if(map){
// row['W'] = map.name
// row['X'] = map.score
// }
// }
// }
//})
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['C'] && row['R']){
def obj = findFirstData(community_CONN, row['C'], row['D'], row['R'])
if(obj){
row['S'] = obj.street_code
}
}
})
def findBestSimilar(town, towns){
def map = [:];
def maxScore = 0
for(def name : towns){
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(town, name);
if(score1>maxScore){
map.name = name
map.score = score1
maxScore = score1
}
}
return map
}
def isDev(development_zone_CONN, def prov, def city, def region, def dev){
def q = new BasicDBObject()
q.put("province", prov)
dev = dev.replace("(","").replace(")","").replace("市","")
if(dev.contains(region) && region.toString().endsWith("区")){
def newRegion = region.replace("区","")
dev = dev.replace(region, newRegion)
}
Pattern pattern = Pattern.compile('^.*' + dev + '.*$', Pattern.CASE_INSENSITIVE);
q.put("name",new BasicDBObject('$regex', pattern))
def rets = development_zone_CONN.find(q).toList()
def developmentZone = null
if(rets){
def devList = []
devList.add(dev);
try{
devList.add(region.replace("县","").replace("区","").replace("市","") + dev);
}catch(e){
println region +" "+ e.getMessage()
}
try{
devList.add(city.replace("市","") + region.replace("县","").replace("区","").replace("市","") + dev);
}catch(e){
println city+" "+region+" "+e.getMessage()
}
try{
devList.add(prov.replace("省","") + city.replace("市","")+ region.replace("县","").replace("区","").replace("市","") + dev);
}catch(e){
println prov+" "+city+" "+region+" "+e.getMessage()
}
try{
devList.add(city.replace("市","") + dev);
}catch(e){
println e
}
try{
devList.add(prov.replace("省","")+ city.replace("市","") + dev);
}catch(e){
println e
}
try{
devList.add(prov.replace("省","")+ region.replace("县","").replace("区","").replace("市","")+ dev);
}catch(e){
println e
}
devList.add(prov.replace("省","")+ dev);
for(def zone : rets){
String name = zone.name.replace("省","").replace("市","").replace("县","")
if(!name.endsWith("区")){
name = name.replace("区","");
}
if(devList.contains(name)){
developmentZone = zone;
break;
}
}
}
if(developmentZone){
return developmentZone.name;
}
return null;
}
def isTown( city, region, town){
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
// obj = it
// },"192.168.0.89")
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
def body = """
{
"query": {
"bool":{
"must":[
{"term": {"city.keyword":"${city}"}},
{"term": {"district.keyword":"${region}"}},
{"term": {"town.keyword":"${town}"}}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
def obj = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
obj = hits[0]
}
}
println(response)
return obj
}
def findCommunityTown(community_CONN, city, region, town){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
Pattern pattern = Pattern.compile(town + '.*$', Pattern.CASE_INSENSITIVE);
q.put("street_name",new BasicDBObject('$regex', pattern))
def rets = community_CONN.find(q).toList()
def obj = null
if(rets){
obj = rets[0]
}
return obj;
}
def findAllTowns(community_CONN, city, region){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
def rets = community_CONN.find(q).toList()
def towns = []
for(def t : rets){
if(!towns.contains(t.street_name)){
towns << t.street_name
}
}
return towns;
}
def findFirstData(community_CONN, city, region, town){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
q.put("street_name", town)
def obj = community_CONN.findOne(q)
return obj
}
def isGov(obj){
String hasGovFlag = null;
if(obj){
hasGovFlag = obj.has_gov;
}
if(StringUtils.isNotBlank(hasGovFlag) && hasGovFlag.equals("是")){
return "是"
}else {
return "否"
}
}
@@ -0,0 +1,53 @@
package com.fangjia.script.gj.village.ailiyun
import com.aliyun.tea.*;
class AnliyunApiUtils {
/**
* 使用AK&SK初始化账号Client
* @param accessKeyId
* @param accessKeySecret
* @return Client
* @throws Exception
*/
public static com.aliyun.iot20180120.Client createClient(String accessKeyId, String accessKeySecret) throws Exception {
com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config()
// 必填,您的 AccessKey ID
.setAccessKeyId(accessKeyId)
// 必填,您的 AccessKey Secret
.setAccessKeySecret(accessKeySecret);
// Endpoint 请参考 https://api.aliyun.com/product/Iot
config.endpoint = "iot.cn-beijing.aliyuncs.com";
return new com.aliyun.iot20180120.Client(config);
}
public static void main(String[] args_) throws Exception {
java.util.List<String> args = java.util.Arrays.asList(args_);
// 请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_ID 和 ALIBABA_CLOUD_ACCESS_KEY_SECRET。
// 工程代码泄露可能会导致 AccessKey 泄露,并威胁账号下所有资源的安全性。以下代码示例使用环境变量获取 AccessKey 的方式进行调用,仅供参考,建议使用更安全的 STS 方式,更多鉴权访问方式请参见:https://help.aliyun.com/document_detail/378657.html
com.aliyun.iot20180120.Client client = createClient(Config.accessKey, Config.accessKeySecret);
com.aliyun.iot20180120.models.CreateDeviceGroupRequest createDeviceGroupRequest = new com.aliyun.iot20180120.models.CreateDeviceGroupRequest()
.setIotInstanceId("your_value")
.setSuperGroupId("your_value")
.setGroupDesc("your_value")
.setGroupName("your_value");
try {
// 复制代码运行请自行打印 API 的返回值
client.createDeviceGroupWithOptions(createDeviceGroupRequest, new com.aliyun.teautil.models.RuntimeOptions());
} catch (TeaException error) {
// 错误 message
System.out.println(error.getMessage());
// 诊断地址
System.out.println(error.getData().get("Recommend"));
com.aliyun.teautil.Common.assertAsString(error.message);
} catch (Exception _error) {
TeaException error = new TeaException(_error.getMessage(), _error);
// 错误 message
System.out.println(error.getMessage());
// 诊断地址
System.out.println(error.getData().get("Recommend"));
com.aliyun.teautil.Common.assertAsString(error.message);
}
}
}
@@ -0,0 +1,14 @@
package com.fangjia.script.gj.village.ailiyun
/* 服务端API签名配置文件
*
* @author: ali
* @version: 0.1 2018-08-08 08:23:54
*/
public class Config {
// AccessKey信息
public static String accessKey = "LTAI5t7UKjXaoGxzet2Vh3bB";
public static String accessKeySecret = "2avjPxmI7FlIRHvMC5C1UuAnd9iG8z";
public final static String CHARSET_UTF8 = "utf8";
}
@@ -0,0 +1,203 @@
package com.fangjia.script.gj.village.ailiyun
import com.aliyuncs.CommonRequest
import com.aliyuncs.CommonResponse
import com.aliyuncs.DefaultAcsClient
import com.aliyuncs.IAcsClient
import com.aliyuncs.exceptions.ClientException
import com.aliyuncs.http.FormatType
import com.aliyuncs.http.MethodType
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import net.sf.json.JSONObject
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.net.URLEncoder
import java.rmi.ServerException;
import java.util.Map;
import java.util.TreeMap;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
/**
* 服务端API签名
*
* @author: ali
* @version: 0.1 2018-06-21 20:47:05
*/
public class SignatureUtils {
private final static String CHARSET_UTF8 = "utf8";
private final static String ALGORITHM = "HmacSHA1";
private final static String SEPARATOR = "&";
public static Map<String, String> splitQueryString(String url)
throws URISyntaxException, UnsupportedEncodingException {
URI uri = new URI(url);
String query = uri.getQuery();
final String[] pairs = query.split("&");
TreeMap<String, String> queryMap = new TreeMap<String, String>();
for (String pair : pairs) {
final int idx = pair.indexOf("=");
final String key = idx > 0 ? pair.substring(0, idx) : pair;
if (!queryMap.containsKey(key)) {
queryMap.put(key, URLDecoder.decode(pair.substring(idx + 1), CHARSET_UTF8));
}
}
return queryMap;
}
public static String generate(String method, Map<String, String> parameter, String accessKeySecret)
throws Exception {
String signString = generateSignString(method, parameter);
System.out.println("signString---" + signString);
byte[] signBytes = hmacSHA1Signature(accessKeySecret + "&", signString);
String signature = newStringByBase64(signBytes);
System.out.println("signature----" + signature);
if ("POST".equals(method))
return signature;
return URLEncoder.encode(signature, "UTF-8");
}
public static String generateSignString(String httpMethod, Map<String, String> parameter) throws IOException {
TreeMap<String, String> sortParameter = new TreeMap<String, String>();
sortParameter.putAll(parameter);
String canonicalizedQueryString = UrlUtil.generateQueryString(sortParameter, true);
if (null == httpMethod) {
throw new RuntimeException("httpMethod can not be empty");
}
StringBuilder stringToSign = new StringBuilder();
stringToSign.append(httpMethod).append(SEPARATOR);
stringToSign.append(percentEncode("/")).append(SEPARATOR);
stringToSign.append(percentEncode(canonicalizedQueryString));
return stringToSign.toString();
}
public static String percentEncode(String value) {
try {
return value == null ? null
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
"~");
} catch (Exception e) {
}
return "";
}
public static byte[] hmacSHA1Signature(String secret, String baseString) throws Exception {
if (StringUtils.isEmpty(secret)) {
throw new IOException("secret can not be empty");
}
if (StringUtils.isEmpty(baseString)) {
return null;
}
Mac mac = Mac.getInstance("HmacSHA1");
SecretKeySpec keySpec = new SecretKeySpec(secret.getBytes(CHARSET_UTF8), ALGORITHM);
mac.init(keySpec);
return mac.doFinal(baseString.getBytes(CHARSET_UTF8));
}
public static String newStringByBase64(byte[] bytes) throws UnsupportedEncodingException {
if (bytes == null || bytes.length == 0) {
return null;
}
return new String(Base64.encodeBase64(bytes, false), CHARSET_UTF8);
}
// public static void main(String[] a){
// Map<String, String> map = new HashMap<String, String>();
// Calendar calendar = Calendar.getInstance()
// def startTime = DateUtil.formatByPattern(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")
// // 公共参数
// map.put("Format", "JSON");
// map.put("Version", "2019-11-18");
// map.put("AccessKeyId", Config.accessKey);
// map.put("SignatureMethod", "HMAC-SHA1");
// map.put("Timestamp", startTime);
// map.put("SignatureVersion", "1.0");
// map.put("SignatureNonce", System.currentTimeMillis().toString());
//// map.put("RegionId", "cn-shanghai");
// // 请求参数
// map.put("Action", "StreetStd");
// map.put("ServiceCode", "addrp");
// map.put("AppKey", "Xczmmbxlur2g");
// map.put("Text","山东省烟台市莱阳市龙门东路1号0027、0028")
// try {
// String signature = SignatureUtils.generate("GET", map, Config.accessKeySecret);
// if(signature){
// map.put("signature", signature);
// }
// String content = new HttpRequest().get("https://address-purification.cn-hangzhou.aliyuncs.com", map, "utf-8")
// JSONObject result = JSONObject.fromObject(content)
// System.out.println("最终signature: " + signature);
// } catch (Exception e) {
// e.printStackTrace();
// }
// System.out.println();
// }
public static def getBaiduAddressStrc(def address) {
DefaultProfile defaultProfile = DefaultProfile.getProfile(
"cn-hangzhou",
Config.accessKey,
Config.accessKeySecret);
IAcsClient client = new DefaultAcsClient(defaultProfile);
// 创建API请求并设置参数
CommonRequest request = new CommonRequest();
request.setSysDomain("address-purification.cn-hangzhou.aliyuncs.com");
request.setSysVersion("2019-11-18");
request.setSysProduct("address-purification");
request.setSysLocationProduct("addrp");
//设置要请求的服务
request.setSysAction("StreetStd");
//设置请求方式
request.setSysMethod(MethodType.POST);
request.setHttpContentType(FormatType.FORM);
//请求头
request.putHeadParameter("x-acs-signature-method", "HMAC-SHA1");
request.putHeadParameter("x-acs-signature-nonce", UUID.randomUUID().toString());
//设置项目token
request.putBodyParameter("AppKey", "Xczmmbxlur2g");
//设置请求参数
request.putBodyParameter("Text", address);
// request.putBodyParameter("DefaultCity", "杭州市");
request.putQueryParameter("AcceptLanguage", "zh-CN");
try {
CommonResponse response = client.getCommonResponse(request);
def data = response.getData();
def result = JSONObject.fromObject(data)
if(result && result.Data){
def data1 = JSONObject.fromObject(result.Data)
def std = data1.street_std
if(std){
def arrs = std.split("\t")
def map = [:]
for(def s : arrs){
def paramValues = s.split("=")
map.put(paramValues[0], paramValues[1])
}
return map
}
}
} catch (ServerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ClientException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
}
@@ -0,0 +1,55 @@
package com.fangjia.script.gj.village.ailiyun
import java.net.URLEncoder;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
/**
* URL处理类
*
* @author: ali
* @version: 0.1 2018-06-21 20:40:52
*/
public class UrlUtil {
private final static String CHARSET_UTF8 = "utf8";
public static String urlEncode(String url) {
if (!StringUtils.isEmpty(url)) {
try {
url = URLEncoder.encode(url, "UTF-8");
} catch (Exception e) {
System.out.println("Url encode error:" + e.getMessage());
}
}
return url;
}
public static String generateQueryString(Map<String, String> params, boolean isEncodeKV) {
StringBuilder canonicalizedQueryString = new StringBuilder();
for (Map.Entry<String, String> entry : params.entrySet()) {
if (isEncodeKV)
canonicalizedQueryString.append(percentEncode(entry.getKey())).append("=")
.append(percentEncode(entry.getValue())).append("&");
else
canonicalizedQueryString.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
}
if (canonicalizedQueryString.length() > 1) {
canonicalizedQueryString.setLength(canonicalizedQueryString.length() - 1);
}
return canonicalizedQueryString.toString();
}
public static String percentEncode(String value) {
try {
// 使用URLEncoder.encode编码后,将"+","*","%7E"做替换即满足API规定的编码规范。
return value == null ? null
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
"~");
} catch (Exception e) {
}
return "";
}
}
@@ -7,6 +7,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
@@ -68,21 +69,24 @@ def filePath = "D:\\
//})
def objs = COMMUNITY_CONN.find(new BasicDBObject())
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
for(def d : objs){
if(d.name){
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
def udata = new BasicDBObject()
udata.put("name_pinyin", py1)
if(d.location){
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
udata.put("location", ["lat":lat, "lon":lon])
}
if(!d.name_pinyin){
if(d.name){
println d._id
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
def udata = new BasicDBObject()
udata.put("name_pinyin", py1)
if(d.location){
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
udata.put("location", ["lat":lat, "lon":lon])
}
def q = new BasicDBObject()
q.put("_id", d._id)
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
def q = new BasicDBObject()
q.put("_id", d._id)
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
}
}
}
@@ -26,7 +26,7 @@ class SuggestionDataSynch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
@@ -57,8 +57,8 @@ class SuggestionDataSynch extends BaseBatch{
}
def getExitsCity(){
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
def cities = SUGGESTION_CONN.distinct("city")
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
def cities = SUGGESTION_CONN.distinct("city", new BasicDBObject("type","poi"))
def simpleCities = []
for (def c :cities) {
def city = RegionData.formatCity(c)
@@ -119,7 +119,7 @@ class SuggestionDataSynch extends BaseBatch{
try{
count = SUGGESTION_CONN.count(q)
}catch(e){
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
count = SUGGESTION_CONN.count(q)
}
// if(count>0){
@@ -46,8 +46,11 @@ class ElasticUtils {
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
connectionProperties.setProperty("user","developer");
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
// connectionProperties.setProperty("user","developer");
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
connectionProperties.setProperty("user","elastic");
connectionProperties.setProperty("password","goojia123456");//elasticsearch
dataSource.setProperties(connectionProperties);
return dataSource;
}
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 5
def nThread = Runtime.getRuntime().availableProcessors() * 3
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)