Merge remote-tracking branch 'origin/master'
This commit is contained in:
+3
-1
@@ -927,7 +927,9 @@ public class HouseController {
|
||||
model.setStartTime(sixMonthsAgoInt);
|
||||
model.setEndTime(todayInt);
|
||||
// 默认3000M内
|
||||
model.setRadius(3000);
|
||||
if (model.getRadius() == null) {
|
||||
model.setRadius(3000);
|
||||
}
|
||||
if (StringUtils.isBlank(model.getLocation())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("location(经纬度)不能为空"));
|
||||
return mav;
|
||||
|
||||
+113
-1
@@ -17,8 +17,10 @@ import com.fangjia.service.api.AmapPoiService;
|
||||
import com.fangjia.service.bo.Point;
|
||||
import com.fangjia.service.datastore.DistrictService;
|
||||
import com.fangjia.service.mongo.fangjia.CbmRegionService;
|
||||
import com.fangjia.service.open.AdministrativeMappingService;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
|
||||
import javax.validation.Valid;
|
||||
import java.util.*;
|
||||
@@ -30,7 +32,11 @@ public class PoiController {
|
||||
private CbmRegionService cbmRegionService;
|
||||
private AddressParserService addressParserService;
|
||||
private DistrictService districtService;
|
||||
|
||||
private AdministrativeMappingService administrativeMappingService;
|
||||
private static final String[] PROPERTY_SEARCH_API_FIELDS = {"address", "city", "region", "name", "addrAlias"};
|
||||
private static final String ES_USER = "developer";
|
||||
private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3";
|
||||
|
||||
public Object getAntiFraudQuestionAction(@Valid AntiFraudQuestionModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
@@ -221,7 +227,7 @@ public class PoiController {
|
||||
public Object searchAction(AddressModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getLocation()) || StringUtils.isBlank(model.getCity())) {
|
||||
if (StringUtils.isBlank(model.getLocation()) || StringUtils.isBlank(model.getCity())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("城市和经纬度为必填项"));
|
||||
return mav;
|
||||
}
|
||||
@@ -268,6 +274,108 @@ public class PoiController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object facilitiesAction(AddressModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
String city = administrativeMappingService.findCityByCityShort(model.getCity());
|
||||
String lngStr = model.getLng();
|
||||
String latStr = model.getLat();
|
||||
String numberPattern = "-?\\d+(\\.\\d+)?"; // 匹配整数或浮点数
|
||||
if (!lngStr.matches(numberPattern) || !latStr.matches(numberPattern)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("location(经纬度)格式错误"));
|
||||
return mav;
|
||||
}
|
||||
double lng = Double.parseDouble(lngStr);
|
||||
double lat = Double.parseDouble(latStr);
|
||||
Map<String, List<String>> result = new HashMap<String, List<String>>();
|
||||
List<String> bankSmallType = Arrays.asList("银行", "中国人民银行", "国家开发银行", "中国进出口银行");
|
||||
List<String> busSmallType = Collections.singletonList("公交车站相关");
|
||||
List<String> hospitalSmallType = Arrays.asList("综合医院", "三级甲等医院", "专科医院");
|
||||
List<String> schoolSmallType = Arrays.asList("学校", "高等院校", "中学", "小学", "幼儿园");
|
||||
List<String> subwaySmallType = Collections.singletonList("地铁站");
|
||||
List<String> supermarketSmallType = Arrays.asList("商场", "购物中心", "普通商场");
|
||||
|
||||
result.put("bank", findAroundByLocation(city, lng, lat, bankSmallType, 1000, true));
|
||||
result.put("bus", findAroundByLocation(city, lng, lat, busSmallType, 1000, false));
|
||||
result.put("hospital", findAroundByLocation(city, lng, lat, hospitalSmallType, 1000, true));
|
||||
result.put("school", findAroundByLocation(city, lng, lat, schoolSmallType, 1000, true));
|
||||
result.put("subway", findAroundByLocation(city, lng, lat, subwaySmallType, 1000, false));
|
||||
result.put("supermarket", findAroundByLocation(city, lng, lat, supermarketSmallType, 1000, true));
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(result));
|
||||
return mav;
|
||||
}
|
||||
|
||||
private List<String> findAroundByLocation(String city, double lng, double lat, List<String> smallTypes, int distance, boolean excludeStatus) {
|
||||
List<String> poiNames = new ArrayList<String>();
|
||||
JSONObject reqBody = new JSONObject();
|
||||
JSONObject queryJson = new JSONObject();
|
||||
List<JSONObject> jsonList = new ArrayList<JSONObject>();
|
||||
List<JSONObject> mustNotList = new ArrayList<JSONObject>();
|
||||
|
||||
if (excludeStatus) {
|
||||
JSONObject statusParam = new JSONObject();
|
||||
statusParam.put("status", 1);
|
||||
JSONObject statusTerm = new JSONObject();
|
||||
statusTerm.put("term", statusParam);
|
||||
mustNotList.add(statusTerm);
|
||||
}
|
||||
JSONObject mainTypeParam = new JSONObject();
|
||||
mainTypeParam.put("main_type", 0);
|
||||
JSONObject mainTypeTerm = new JSONObject();
|
||||
mainTypeTerm.put("term", mainTypeParam);
|
||||
mustNotList.add(mainTypeTerm);
|
||||
|
||||
JSONObject cityParam = new JSONObject();
|
||||
cityParam.put("city", city);
|
||||
JSONObject cityTerm = new JSONObject();
|
||||
cityTerm.put("term", cityParam);
|
||||
jsonList.add(cityTerm);
|
||||
|
||||
JSONObject smallTypeParam = new JSONObject();
|
||||
smallTypeParam.put("small_type", smallTypes);
|
||||
JSONObject smallTypeTerm = new JSONObject();
|
||||
smallTypeTerm.put("terms", smallTypeParam);
|
||||
jsonList.add(smallTypeTerm);
|
||||
|
||||
JSONObject geoDistance = new JSONObject();
|
||||
geoDistance.put("distance", distance + "m");
|
||||
geoDistance.put("location", Arrays.asList(lng, lat));
|
||||
JSONObject geoQuery = new JSONObject();
|
||||
geoQuery.put("geo_distance", geoDistance);
|
||||
jsonList.add(geoQuery);
|
||||
|
||||
queryJson.put("must", jsonList);
|
||||
if (!mustNotList.isEmpty()) {
|
||||
queryJson.put("must_not", mustNotList);
|
||||
}
|
||||
JSONObject boolQuery = new JSONObject();
|
||||
boolQuery.put("bool", queryJson);
|
||||
reqBody.put("query", boolQuery);
|
||||
|
||||
Base64 token = new Base64();
|
||||
String authEncoding = token.encodeAsString((ES_USER + ":" + ES_PASSWORD).getBytes());
|
||||
HttpRequest request = new HttpRequest();
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Basic " + authEncoding);
|
||||
|
||||
try {
|
||||
String response = request.postWithBody("http://192.168.0.89:9200/gaode_map_poi/_search?rest_total_hits_as_int=true", reqBody, headers, "UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(response);
|
||||
List<Map<String, Object>> hits = (List<Map<String, Object>>) json.getJSONObject("hits").get("hits");
|
||||
|
||||
for (Map<String, Object> hit : hits) {
|
||||
Map<String, Object> source = (Map<String, Object>) hit.get("_source");
|
||||
if (source.containsKey("name")) {
|
||||
poiNames.add(source.get("name").toString());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return poiNames;
|
||||
}
|
||||
|
||||
public void setAmapPoiService(AmapPoiService amapPoiService) {
|
||||
this.amapPoiService = amapPoiService;
|
||||
}
|
||||
@@ -283,4 +391,8 @@ public class PoiController {
|
||||
public void setDistrictService(DistrictService districtService) {
|
||||
this.districtService = districtService;
|
||||
}
|
||||
|
||||
public void setAdministrativeMappingService(AdministrativeMappingService administrativeMappingService) {
|
||||
this.administrativeMappingService = administrativeMappingService;
|
||||
}
|
||||
}
|
||||
|
||||
+9
-1
@@ -146,12 +146,20 @@ public class DistrictController {
|
||||
if (!district.getName().contains(keyword)) {
|
||||
if (district.getAlias() != null && !district.getAlias().isEmpty()) {
|
||||
for (String alias : district.getAlias()) {
|
||||
if (alias.contains(keyword)){
|
||||
if (alias.contains(keyword)) {
|
||||
bo.setAlias(alias);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (district.getAddrAlias() != null && !district.getAddrAlias().isEmpty()) {
|
||||
for (String addrAlias : district.getAddrAlias()) {
|
||||
if (addrAlias.contains(keyword)) {
|
||||
bo.setAddrAlias(addrAlias);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ret.add(bo);
|
||||
}
|
||||
|
||||
@@ -909,14 +909,18 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
static def getNumber(def input) {
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
try{
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
}
|
||||
}catch(Exception e){
|
||||
println input+" error"
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 根据链家数据修改小区库中字段
|
||||
* 使用到中间映射表
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class UpdateDistrictField extends BaseBatch{
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
|
||||
|
||||
def sources = ["链家", "我爱我家", "安居客"]
|
||||
|
||||
def statMap = [:]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new UpdateDistrictField(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def sourceMap = [:]
|
||||
sources.each {source->
|
||||
def valMap = findThirdMatchDist(source)
|
||||
println source +" "+valMap.size()
|
||||
if(valMap){
|
||||
sourceMap.put(source, valMap)
|
||||
}
|
||||
}
|
||||
def margeResults = megerDatas(sourceMap)
|
||||
updateDB(margeResults)
|
||||
println "修改完成 "+new Date().toString()
|
||||
}
|
||||
|
||||
def megerDatas(sourceMap){
|
||||
def set = new HashSet()
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
set.addAll(valMap.keySet())
|
||||
}
|
||||
def lastValues = []
|
||||
set.each {id->
|
||||
|
||||
def dbObj = CommonDatautils.findDistrictById(id)
|
||||
|
||||
def datas = []
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
def sourceEsObj = findCommunities(valMap.get(id))
|
||||
if(sourceEsObj){
|
||||
if(source == "链家"){
|
||||
sourceEsObj.source = "lianjia"
|
||||
}else if(source == "我爱我家"){
|
||||
sourceEsObj.source = "wiwj"
|
||||
}else if(source == "安居客"){
|
||||
sourceEsObj.source = "anjuke"
|
||||
}
|
||||
datas << sourceEsObj
|
||||
}
|
||||
}
|
||||
if(dbObj && datas){
|
||||
def newValMap = getNewField2(datas, dbObj)
|
||||
if(newValMap){
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
statMap.each {source,fieldMap->
|
||||
println source +" "+fieldMap.size()
|
||||
fieldMap.each{f,v->
|
||||
println f+" "+ v
|
||||
}
|
||||
println "----------------------------------------"
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
|
||||
|
||||
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||
// def lianjisIds = lianjiaMap.keySet()
|
||||
// def wiwjIds = wiwjMap.keySet()
|
||||
// def anjukeIds = anjukeMap.keySet()
|
||||
// def set = new HashSet()
|
||||
// set.addAll(lianjisIds)
|
||||
// set.addAll(wiwjIds)
|
||||
// set.addAll(anjukeIds)
|
||||
//
|
||||
// def lastValues = []
|
||||
// def lianjiaUpdateCount = 0
|
||||
// def wiwjUpdateCount = 0
|
||||
// def anjukeUpdateCount = 0
|
||||
// def lianjiaCountMap = [:]
|
||||
// def wiwjCountMap = [:]
|
||||
// def anjukeCountMap = [:]
|
||||
// set.each {id->
|
||||
// BasicDBObject newValMap = new BasicDBObject()
|
||||
// fields.each {field->
|
||||
// def lianjiaValMap = lianjiaMap.get(id)
|
||||
// def wiwjValMap = wiwjMap.get(id)
|
||||
// def anjukeValMap = anjukeMap.get(id)
|
||||
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||
// newValMap.put(field, lianjiaValMap.get(field))
|
||||
// lianjiaUpdateCount++
|
||||
// def count = lianjiaCountMap.get(field)
|
||||
// if(!count){
|
||||
// lianjiaCountMap.put(field, 1)
|
||||
// }else{
|
||||
// lianjiaCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||
// newValMap.put(field, wiwjValMap.get(field))
|
||||
// wiwjUpdateCount++
|
||||
// def count = wiwjCountMap.get(field)
|
||||
// if(!count){
|
||||
// wiwjCountMap.put(field, 1)
|
||||
// }else{
|
||||
// wiwjCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||
// newValMap.put(field, anjukeValMap.get(field))
|
||||
// anjukeUpdateCount++
|
||||
// def count = anjukeCountMap.get(field)
|
||||
// if(!count){
|
||||
// anjukeCountMap.put(field, 1)
|
||||
// }else{
|
||||
// anjukeCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// newValMap.put("id", id)
|
||||
// lastValues << newValMap
|
||||
// }
|
||||
//
|
||||
// lianjiaCountMap.each {k,v->
|
||||
// println "[链家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// wiwjCountMap.each {k,v->
|
||||
// println "[我爱我家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// anjukeCountMap.each {k,v->
|
||||
// println "[安居客] "+k+" "+v
|
||||
// }
|
||||
// return lastValues
|
||||
// }
|
||||
|
||||
def updateDB(margeResults){
|
||||
for(def mr : margeResults){
|
||||
def id = mr.get("id")
|
||||
mr.remove("id")
|
||||
mr.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
|
||||
}
|
||||
}
|
||||
|
||||
// def batchUpdateDist(source){
|
||||
//// def rets = findThirdMatchDist(source)
|
||||
// println "[${source}] 符保条件小区量:"+rets.size()
|
||||
// def updateIndex = new AtomicLong()
|
||||
// final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
// def allUpdateList = new CopyOnWriteArrayList()
|
||||
//
|
||||
// def sourceMap = new ConcurrentHashMap()
|
||||
//
|
||||
// for(def i=0;i<rets.size();i++) {
|
||||
// final def ret = rets.get(i)
|
||||
//// fixedThreadPool.execute(
|
||||
//// new Runnable() {
|
||||
//// void run() {
|
||||
// try {
|
||||
// def newValMap = running(ret.url, ret.old_match_id)
|
||||
// if(newValMap){
|
||||
// sourceMap.put(ret.old_match_id, newValMap)
|
||||
// allUpdateList << newValMap
|
||||
// updateIndex.addAndGet(1)
|
||||
// }
|
||||
// }catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
//// }
|
||||
//// }
|
||||
//// )
|
||||
// }
|
||||
// latch.await()
|
||||
// println "总修改量:"+updateIndex.get()
|
||||
// def countMap = [:]
|
||||
// for(def map : allUpdateList){
|
||||
// map.each{k,v->
|
||||
// def count = countMap.get(k)
|
||||
// if(!count){
|
||||
// countMap.put(k, 1)
|
||||
// }else{
|
||||
// countMap.put(k, count+1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// countMap.each {k,v->
|
||||
// println k +" 更新量:"+v
|
||||
// }
|
||||
//
|
||||
// println "---------------------------------------------------------"
|
||||
// return sourceMap
|
||||
// }
|
||||
|
||||
def running(url, old_match_id){
|
||||
def esObj = findCommunities(url)
|
||||
def dbObj = CommonDatautils.findDistrictById(old_match_id)
|
||||
if(dbObj){
|
||||
def newValMap = getNewField(esObj, dbObj)
|
||||
return newValMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getNewField(esObj, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
esObj.each{field, esVal->
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(esVal && field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
|
||||
}
|
||||
if(esVal && field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(esVal && field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
newValMap.put(field, newVal)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return newValMap
|
||||
}
|
||||
|
||||
def getNewField2(esDatas, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
def lastValueMap = [:]
|
||||
def haveValueMap = [:]
|
||||
fields.each{field->
|
||||
for(def esData : esDatas){
|
||||
def esVal = esData.get(field)
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(haveValueMap.containsKey(field)){
|
||||
continue
|
||||
}else{
|
||||
if(lastValueMap.containsKey(field)){
|
||||
continue
|
||||
}
|
||||
if(esVal){
|
||||
|
||||
if(field=="estate_charge" || field=="car_site_count") {
|
||||
esVal = FiledStrand.getNumber(esVal)
|
||||
}
|
||||
if(dbVal && esVal.toString()==dbVal.toString()){
|
||||
haveValueMap.put(field, esVal)
|
||||
}else{
|
||||
if(field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
lastValueMap.put(field, newVal)
|
||||
|
||||
//以下为统计各网站各字段的更新情况
|
||||
def fieldStatMap = statMap.get(esData.source)
|
||||
if(!fieldStatMap){
|
||||
fieldStatMap = [:]
|
||||
statMap.put(esData.source, fieldStatMap)
|
||||
}
|
||||
def count = fieldStatMap.get(field)
|
||||
if(!count){
|
||||
fieldStatMap.put(field, 1)
|
||||
}else{
|
||||
fieldStatMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValueMap
|
||||
}
|
||||
|
||||
def findThirdMatchDist(source){
|
||||
def query = new BasicDBObject("source",source).append("old_match_way", 1)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("url", 1)
|
||||
keys.put("old_match_id", 1)
|
||||
keys.put("old_match_city", 1)
|
||||
keys.put("old_match_district", 1)
|
||||
keys.put("old_match_name", 1)
|
||||
def idsMap = [:]
|
||||
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
|
||||
for(def db : dbcursor){
|
||||
if(!db.old_match_id && db.old_match_name){
|
||||
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
|
||||
if(dbo){
|
||||
idsMap.put(dbo._id.toString(), db.url)
|
||||
}
|
||||
}else{
|
||||
idsMap.put(db.old_match_id, db.url)
|
||||
}
|
||||
|
||||
}
|
||||
return idsMap
|
||||
}
|
||||
|
||||
static def findCommunities(url){
|
||||
if(!url){
|
||||
return null
|
||||
}
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||
def rets = []
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
}catch(Exception e){
|
||||
findCommunities(url)
|
||||
}
|
||||
def data = [:]
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
data.put("household_count", d.households)
|
||||
data.put("loudong_count", d.loudong_cnt)
|
||||
data.put("developer", d.developer)
|
||||
data.put("estate_company", d.property_company)
|
||||
data.put("estate_charge", d.property_fee)
|
||||
data.put("car_site_count", d.parking_cnt)
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
+28
-21
@@ -38,7 +38,7 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
def citys = CommonUtils.findAllCity()
|
||||
citys.each {city->
|
||||
@@ -48,7 +48,7 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
matchCommunitys(communitys)
|
||||
}
|
||||
|
||||
// statImgCount()
|
||||
statImgCount()
|
||||
}
|
||||
def findAllEsData2(city, mdate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||
@@ -105,40 +105,47 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
def total = communitys.size()
|
||||
for(def i=0;i<communitys.size();i++){
|
||||
final def d = communitys[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
|
||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
||||
def obj = thirdSeaweedMappingColl.findOne(query)
|
||||
if(obj) {
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images){
|
||||
vo.put("out_pics", d.out_images)
|
||||
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
|
||||
if(obj.old_match_id && obj.match_id==obj.old_match_id){
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images && (!imageObj || (imageObj && !imageObj.out_pics))){
|
||||
vo.put("out_pics", d.out_images)
|
||||
}
|
||||
if(d.house_type_images && (!imageObj || (imageObj && !imageObj.floorplan_pics))){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
if(vo.out_pics || vo.floorplan_pics){
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}
|
||||
}
|
||||
if(d.house_type_images){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}else{
|
||||
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
});
|
||||
}
|
||||
// latch.await()
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
+1
@@ -1062,6 +1062,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
q.put("s_date", SHORT_DATE);
|
||||
|
||||
DBObject udata2 = new BasicDBObject()
|
||||
udata2.put("match_id", district._id.toString());
|
||||
udata2.put("match_city", district.city);
|
||||
udata2.put("match_region", district.region);
|
||||
udata2.put("match_name", district.name);
|
||||
|
||||
@@ -24,6 +24,8 @@ public class AutoCompleteBo {
|
||||
|
||||
private String alias;
|
||||
|
||||
private String addrAlias;
|
||||
|
||||
/*** @Deprecated end ****/
|
||||
|
||||
public String getName() {
|
||||
@@ -145,4 +147,12 @@ public class AutoCompleteBo {
|
||||
public void setAlias(String alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
|
||||
public String getAddrAlias() {
|
||||
return addrAlias;
|
||||
}
|
||||
|
||||
public void setAddrAlias(String addrAlias) {
|
||||
this.addrAlias = addrAlias;
|
||||
}
|
||||
}
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.dao.AdministrativeMappingDao;
|
||||
import com.fangjia.dal.mongo.model.AdministrativeMapping;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public interface AdministrativeMappingService {
|
||||
|
||||
|
||||
|
||||
public String findCityCodeByCityShort(String cityShort);
|
||||
public String findCityByCityShort(String cityShort) ;
|
||||
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.dao.AdministrativeMappingDao;
|
||||
import com.fangjia.dal.mongo.model.AdministrativeMapping;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class AdministrativeMappingServiceImpl implements AdministrativeMappingService {
|
||||
private AdministrativeMappingDao administrativeMappingDao;
|
||||
private List<String> CITY_SHORT = new ArrayList<String>(Arrays.asList(
|
||||
"五家渠", "图木舒克", "石河子", "吐鲁番", "中卫", "石嘴山", "海北州",
|
||||
"张掖", "天水", "铜川", "山南", "昌都", "拉萨", "咸阳", "迪庆州",
|
||||
"楚雄", "白银", "榆林", "普洱", "丽江", "昆明", "玉树州", "甘孜州",
|
||||
"阿坝州", "银川", "雅安", "文山州", "西双版纳州", "琼中", "保亭", "乐东",
|
||||
"昌江", "白沙", "临高", "陇南", "定安", "万宁", "宝鸡", "三亚", "来宾",
|
||||
"贺州", "怒江州", "河池", "贵港", "防城港", "梧州", "柳州", "中山", "清远",
|
||||
"定西", "临沧", "德宏州", "琼海", "崇左", "鹤岗", "安阳", "宿迁",
|
||||
"泰州", "盐城", "齐齐哈尔", "淮安", "澄迈", "营口", "宿州", "甘南", "泰安",
|
||||
"无锡", "鄂州", "吴忠", "玉林", "鄂尔多斯", "湖州", "辽源", "邯郸", "六安",
|
||||
"海口", "双鸭山", "抚顺", "白城", "贵阳", "运城", "徐州", "蚌埠",
|
||||
"白山", "延安", "通化", "牡丹江", "临汾", "云浮", "朝阳", "保山",
|
||||
"铁岭", "淮南", "金华", "长治", "克孜勒苏", "四平", "太原", "梅州",
|
||||
"北京", "阜新", "晋中", "阿拉善盟", "晋城", "石家庄", "五指山", "鸡西",
|
||||
"抚州", "果洛州", "驻马店", "芜湖", "呼和浩特", "邢台", "黑河", "七台河",
|
||||
"承德", "龙岩", "镇江", "苏州", "衡水", "安康", "南宁", "包头",
|
||||
"赤峰", "济源", "绍兴", "张家口", "大同", "阳泉", "吕梁", "乌海",
|
||||
"河源", "通辽", "滁州", "东莞", "杭州", "巴彦淖尔", "常州", "乌兰察布",
|
||||
"新余", "延边", "九江", "临夏", "湘西", "保定", "林芝", "沧州",
|
||||
"阜阳", "临沂", "乌鲁木齐", "锡林郭勒盟", "本溪", "德州", "扬州",
|
||||
"鞍山", "松原", "丽水", "衢州", "合肥", "屯昌", "黄山", "武威",
|
||||
"亳州", "宣城", "三明", "绥化", "湘潭", "衡阳", "南平", "沈阳",
|
||||
"辽阳", "南昌", "景德镇", "鹰潭", "赣州", "兴安盟", "吉安",
|
||||
"庆阳", "宜春", "上饶", "潍坊", "济南", "淄博", "哈尔滨",
|
||||
"枣庄", "忻州", "济宁", "日照", "韶关", "开封", "商丘", "大理",
|
||||
"菏泽", "西宁", "郑州", "神农架", "平顶山", "朔州", "鹤壁",
|
||||
"新乡", "东方", "焦作", "伊春", "萍乡", "常德", "周口",
|
||||
"渭南", "张家界", "阿里", "桂林", "大庆", "长沙", "益阳",
|
||||
"株洲", "邵阳", "郴州", "永州", "娄底", "胡杨河", "昆玉",
|
||||
"嘉峪关", "北屯", "塔城", "巴音郭楞", "博尔塔拉", "昌吉",
|
||||
"阿拉尔", "克拉玛依", "海西州", "黄南州", "酒泉", "平凉",
|
||||
"汉中", "昭通", "黔西南", "铜仁", "遵义", "资阳", "巴中",
|
||||
"广安", "宜宾", "眉山", "南充", "毕节", "乐山", "达州",
|
||||
"遂宁", "广元", "绵阳", "德阳", "伊犁", "成都", "安顺",
|
||||
"重庆", "陵水", "红河", "可克达拉", "文昌", "儋州", "百色",
|
||||
"北海", "揭阳", "潮州", "阳江", "汕尾", "双河", "肇庆",
|
||||
"湛江", "汕头", "深圳", "内江", "广州", "台州", "嘉兴",
|
||||
"茂名", "宁波", "三沙", "南通", "和田", "舟山", "连云港",
|
||||
"喀什", "南京", "锦州", "长春", "固原", "温州", "上海",
|
||||
"佛山", "钦州", "六盘水", "潜江", "葫芦岛", "岳阳", "曲靖",
|
||||
"佳木斯", "大兴安岭", "濮阳", "信阳", "烟台", "安庆",
|
||||
"秦皇岛", "天津", "廊坊", "黔南", "那曲", "海南州",
|
||||
"日喀则", "武汉", "怀化", "三门峡", "海东", "兰州",
|
||||
"聊城", "凉山州", "泸州", "南阳", "黔东南", "吉林",
|
||||
"阿克苏", "滨州", "唐山", "铜陵", "金昌", "丹东",
|
||||
"玉溪", "白杨", "呼伦贝尔", "铁门关", "仙桃", "随州",
|
||||
"淮北", "商洛", "江门", "池州", "盘锦", "厦门",
|
||||
"西安", "自贡", "莆田", "泉州", "漳州", "福州",
|
||||
"马鞍山", "宁德", "大连", "孝感", "十堰", "青岛",
|
||||
"攀枝花", "东营", "威海", "新星", "洛阳", "黄石",
|
||||
"阿勒泰", "许昌", "漯河", "宜昌", "哈密", "襄阳",
|
||||
"荆门", "荆州", "黄冈", "恩施", "天门", "咸宁",
|
||||
"惠州", "珠海"
|
||||
));
|
||||
|
||||
|
||||
public String findCityCodeByCityShort(String cityShort) {
|
||||
ArrayList<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("cityShort", cityShort));
|
||||
params.add(new QueryParam("type", "城市"));
|
||||
AdministrativeMapping firstByParams = administrativeMappingDao.findFirstByParams(params);
|
||||
if (firstByParams != null && StringUtils.isNotBlank(firstByParams.getMzbCityCode())) {
|
||||
return firstByParams.getMzbCityCode();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public String findCityByCityShort(String cityShort) {
|
||||
if (CITY_SHORT.contains(cityShort)) {
|
||||
ArrayList<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("cityShort", cityShort));
|
||||
params.add(new QueryParam("type", "城市"));
|
||||
AdministrativeMapping firstByParams = administrativeMappingDao.findFirstByParams(params);
|
||||
if (firstByParams != null && StringUtils.isNotBlank(firstByParams.getMzbCityCode())) {
|
||||
return firstByParams.getCity();
|
||||
}
|
||||
}
|
||||
return cityShort;
|
||||
}
|
||||
|
||||
public void setAdministrativeMappingDao(AdministrativeMappingDao administrativeMappingDao) {
|
||||
this.administrativeMappingDao = administrativeMappingDao;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user