Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -29,7 +29,7 @@
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-core</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>0.2.0</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -154,7 +154,7 @@
|
||||
<dependency>
|
||||
<groupId>com.poseidon</groupId>
|
||||
<artifactId>poseidon-message</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<version>0.0.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.mongodb</groupId>
|
||||
|
||||
@@ -99,7 +99,7 @@
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-web</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>0.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
|
||||
+9
@@ -12,6 +12,7 @@ import com.fangjia.service.mongo.fangjia.SeawaterService;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.sothis.core.util.Pager;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.Arrays;
|
||||
@@ -116,6 +117,14 @@ public class SeawaterController {
|
||||
return ret;
|
||||
}
|
||||
|
||||
public Object batchDeleteAction(@Param(name="ids") String ids) throws UnsupportedEncodingException {
|
||||
JsonModel ret = new JsonModel();
|
||||
if (StringUtils.isNotBlank(ids)){
|
||||
seawaterService.deleteByIds(ids);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public void setSeawaterService(SeawaterService seawaterService) {
|
||||
this.seawaterService = seawaterService;
|
||||
}
|
||||
|
||||
@@ -118,9 +118,13 @@
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
|
||||
<div>
|
||||
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
|
||||
<input class ="btn btn-primary" id="batchDelete" type="submit" value="批量删除" style="float: right" />
|
||||
</div>
|
||||
<table id="dataSheet" class="table table-hover table-condensed" name="reginTable">
|
||||
<tr>
|
||||
<th style="width:30px;"><input type="checkbox" id="checkAll"/>全选</th>
|
||||
<th style="width:60px;">类型</th>
|
||||
<th style="width:50px;">区域</th>
|
||||
<th style="width:130px;">小区名</th>
|
||||
@@ -142,6 +146,7 @@
|
||||
<#if seawaterList??>
|
||||
<#list seawaterList as it>
|
||||
<tr>
|
||||
<td><input type="checkbox" name="houseCheck" value="${it.id}"/></td>
|
||||
<td>
|
||||
<#if it.category! == 'sell'>
|
||||
二手房
|
||||
@@ -249,7 +254,7 @@
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
changeCity('${cityDisplay!}');
|
||||
|
||||
$(".hrefClick").click(function(){
|
||||
@@ -336,8 +341,48 @@
|
||||
$("#roomSpan").hide();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 全选
|
||||
$("#checkAll").click(function(){
|
||||
$('input[name="houseCheck"]').each(function() {
|
||||
var _this = this
|
||||
if (_this.checked == true) {
|
||||
_this.checked = false
|
||||
} else {
|
||||
_this.checked = true
|
||||
}
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
$("#batchDelete").click(function(){
|
||||
if(confirm('确认要删除所选的数据吗?')) {
|
||||
var ids = "";
|
||||
$('input:checkbox[name=houseCheck]:checked').each(function() {
|
||||
var _this = $(this);
|
||||
ids += _this.val() + ","
|
||||
});
|
||||
if (ids == "") {
|
||||
alert("请选择数据")
|
||||
return false;
|
||||
}
|
||||
jQuery.ajax({
|
||||
type: 'POST',
|
||||
url: '../seawater/batchDelete',
|
||||
data:{ids:ids},
|
||||
dataType: 'json',
|
||||
timeout: 100000,
|
||||
success: function(model) {
|
||||
alert("批量删除操作成功");
|
||||
location.reload();
|
||||
},
|
||||
error : function() {
|
||||
//alert("操作失败");
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
</script>
|
||||
</html>
|
||||
</#compress>
|
||||
@@ -205,7 +205,6 @@ public abstract class ElasticDaoImpl<E extends MongoEntity> extends AbstractJpaC
|
||||
|
||||
@Override
|
||||
public void deleteByIds(List<String> idList) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -14,10 +14,8 @@ public interface SeawaterDao extends com.fangjia.dal.SeawaterDao<Seawater> {
|
||||
|
||||
Seawater findByIdFromEs(String id,String table);
|
||||
|
||||
|
||||
List<Seawater> findSeawatersFromEs(String category, String city, String region, String name, List<String> alias, List<String> regionAlias, Integer startDate, Integer endDate,
|
||||
Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root, String sort, boolean order, int start, int limit , String table);
|
||||
|
||||
List<Seawater> findHouses(String city, String region, List<String> alias, Integer minPrice, Integer maxPrice, Integer limit);
|
||||
|
||||
}
|
||||
|
||||
@@ -96,6 +96,44 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
return new ArrayList<Seawater>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteById(String id) {
|
||||
deleteByIds(Arrays.asList(id));
|
||||
}
|
||||
|
||||
private Map<String, String> buildHeader() {
|
||||
Base64 token = new Base64();
|
||||
String authenticationEncoding = token.encodeAsString(new String("" + ES_USER + ":" + ES_PASSWORD + "").getBytes());
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Basic " + authenticationEncoding + "");
|
||||
return headers;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByIds(List<String> idList) {
|
||||
HttpRequest request = new HttpRequest();
|
||||
|
||||
JSONObject reqBody = new JSONObject();
|
||||
JSONObject query = new JSONObject();
|
||||
JSONObject bool = new JSONObject();
|
||||
JSONArray must = new JSONArray();
|
||||
must.add(addTerms("_id", idList));
|
||||
bool.put("must", must);
|
||||
query.put("bool", bool);
|
||||
reqBody.put("query", query);
|
||||
|
||||
JSONObject script = new JSONObject();
|
||||
script.put("source", "ctx._source.status=1");
|
||||
reqBody.put("script", script);
|
||||
|
||||
try {
|
||||
request.postWithBody("http://es.fangjia.com:9200/seawater/_update_by_query", reqBody, this.buildHeader(), "UTF-8");
|
||||
} catch (HttpRequestException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject addMatch(String name, String value){
|
||||
JSONObject term = new JSONObject();
|
||||
JSONObject match = new JSONObject();
|
||||
@@ -338,6 +376,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
if(limit >= 0){
|
||||
reqBody.put("size", limit);
|
||||
}
|
||||
|
||||
// ²»Õ¹Ê¾É¾³ý
|
||||
JSONObject param = new JSONObject();
|
||||
param.put("status", 0);
|
||||
JSONObject temp = new JSONObject();
|
||||
temp.put("term", param);
|
||||
jsonList.add(temp);
|
||||
|
||||
query.put("must", jsonList);
|
||||
q.put("bool", query);
|
||||
reqBody.put("query", q);
|
||||
|
||||
+1
@@ -1790,6 +1790,7 @@ public class AddressController {
|
||||
}
|
||||
}
|
||||
dto.append("buildingNumber", address.getBuildingNumber());
|
||||
dto.append("floor", address.getFloor());
|
||||
dto.append("unitNumber", address.getUnitNumber());
|
||||
dto.append("roomNumber", address.getRoomNumber());
|
||||
dto.append("formattedAddress", address.getFormattedAddress());
|
||||
|
||||
+1
-1
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "工商大楼");
|
||||
// q.put("name","东寓新村")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
|
||||
+9
-9
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
|
||||
houseLoudng.fitment_cost = ret.fitment_charge
|
||||
houseLoudng.car_port = ret.carport
|
||||
houseLoudng.house_height = ret.house_hight
|
||||
houseLoudng.worry_facility" : null,
|
||||
houseLoudng.fitment_age = ret.fitment_age
|
||||
houseLoudng.lighting_ventilation" : null,
|
||||
houseLoudng.build_type = ret.build_type
|
||||
houseLoudng.house_property" : null,
|
||||
houseLoudng.noise" : null,
|
||||
houseLoudng.sight" : null,
|
||||
houseLoudng.fitment_rank" : null,
|
||||
houseLoudng.sideset" : null,
|
||||
// houseLoudng.worry_facility" : null,
|
||||
// houseLoudng.fitment_age = ret.fitment_age
|
||||
// houseLoudng.lighting_ventilation" : null,
|
||||
// houseLoudng.build_type = ret.build_type
|
||||
// houseLoudng.house_property" : null,
|
||||
// houseLoudng.noise" : null,
|
||||
// houseLoudng.sight" : null,
|
||||
// houseLoudng.fitment_rank" : null,
|
||||
// houseLoudng.sideset" : null,
|
||||
|
||||
|
||||
houseLoudng.district_no = district._id
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,117 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class LogStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\建行匹配小区2.xlsx", 0, 1)
|
||||
println datas.size()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行匹配小区2.xlsx",{ d ->
|
||||
def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
|
||||
def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
|
||||
if(dbo){
|
||||
d['E'] = dbo.avg_price
|
||||
}
|
||||
|
||||
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(district){
|
||||
def startDate = DateUtil.addMonth(calDate, -6)
|
||||
def start = startDate.replace("-","").toInteger()
|
||||
def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
|
||||
d['G'] = count
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
|
||||
String sql = "select * from seawater-20* where cat=sell and city='${city}' and region='${region}' and district_name in (${alias.join(',')})"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
rets << data
|
||||
println data
|
||||
})
|
||||
return rets.size()
|
||||
}
|
||||
|
||||
// def findTestDatas(def city, def value) {
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("user_id", "1007751")
|
||||
// q.put("parameter_map.city", city)
|
||||
// q.put("request_uri", "/property/evaluateForTest")
|
||||
//
|
||||
// def keys = new BasicDBObject()
|
||||
// keys.put("res_map",1)
|
||||
// def cursor = LOG.find(q)
|
||||
// def count = 0
|
||||
// println "总量:"+cursor.count()
|
||||
// def datas = []
|
||||
// for(def d : cursor){
|
||||
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
|
||||
// def resMap = d.res_map.result
|
||||
// if(resMap.calcType==value){
|
||||
// def data = []
|
||||
// def rowMap = [:]
|
||||
// rowMap.city = resMap.city
|
||||
// rowMap.region = resMap.district
|
||||
// rowMap.name = resMap.name
|
||||
// rowMap.calDate = d.parameter_map.calDate
|
||||
// data << rowMap
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return datas;
|
||||
// }
|
||||
|
||||
// for(def i=0;i<datas.size();i++){
|
||||
// def d = datas[i];
|
||||
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
//
|
||||
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// if(district){
|
||||
// def startDate = DateUtil.addMonth(calDate, -6)
|
||||
// def start = startDate.replace("-","").toInteger()
|
||||
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
// def alias = []
|
||||
// if(district.alias){
|
||||
// if(!district.alias.contains(district.name)){
|
||||
// district.alias << district.name
|
||||
// }
|
||||
// alias = district.alias
|
||||
// }else{
|
||||
// alias << district.name
|
||||
// }
|
||||
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
|
||||
// d['G'] = count
|
||||
// }
|
||||
|
||||
|
||||
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.gj.address
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AddressParser {
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddressParser();
|
||||
task.run()
|
||||
// task.gaoDeAddress()
|
||||
}
|
||||
|
||||
def gaoDeAddress(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match2(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def match(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
// if(region && !name.contains(region)){
|
||||
// locationMap = matchLocation(city, region + name, "name")
|
||||
// }else{
|
||||
locationMap = matchLocation(city + region, name, "name")
|
||||
// }
|
||||
// if(!locationMap){
|
||||
// locationMap = matchLocation(city, address, "address")
|
||||
// }
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("school_lat1", locationMap.glat)
|
||||
data.put("school_lng1", locationMap.glng)
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def match2(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
|
||||
locationMap = matchLocation2(city + region, name, "name")
|
||||
|
||||
if(!locationMap){
|
||||
locationMap = matchLocation2(city, address, "address")
|
||||
}
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("gaode_name", locationMap.name)
|
||||
data.put("gaode_address", locationMap.address)
|
||||
if(locationMap.location){
|
||||
def arrs = locationMap.location.split(",")
|
||||
data.put("school_lat", arrs[1].toDouble())
|
||||
data.put("school_lng", arrs[0].toDouble())
|
||||
}
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
def matchLocation(city, address, cat){
|
||||
|
||||
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def datas = geocodes.get(0)
|
||||
if(datas.formatted_address && cat=="name"){
|
||||
if(datas.formatted_address.contains(address)){
|
||||
def glocation = datas.location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}else{
|
||||
return null
|
||||
}
|
||||
}else{
|
||||
def glocation = geocodes.get(0).location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def matchLocation2(city, address, cat){
|
||||
|
||||
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("pois")) {
|
||||
def geocodes = gloJson.get("pois")
|
||||
def datas = geocodes.get(0)
|
||||
return datas
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("school_name","苏州市娄葑实验小学")
|
||||
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DistrictSchoolDataExport {
|
||||
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def regionCodeMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSchoolDataExport()
|
||||
// task.run()
|
||||
task.megre()
|
||||
}
|
||||
|
||||
def megre(){
|
||||
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
|
||||
def ids = []
|
||||
for(def d : datas1){
|
||||
ids << d['U']
|
||||
}
|
||||
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
|
||||
for(def d : datas2){
|
||||
def id = d['U']
|
||||
if(!ids.contains(id)){
|
||||
datas1 << d
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "苏州"
|
||||
getRegionCodeByCity(city)
|
||||
getRegionCodeByCity("合肥")
|
||||
getRegionCodeByCity("西安")
|
||||
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts = []
|
||||
districts.addAll(districts0)
|
||||
districts.addAll(districts1)
|
||||
districts.addAll(districts2)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
|
||||
def i = 0
|
||||
for(def k=0;k<districts.size();k++){
|
||||
final def dist = districts.get(k)
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
|
||||
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
|
||||
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
|
||||
if(primarySchools || midSchools){
|
||||
def info1 = null;
|
||||
def info2 = null;
|
||||
def distance1 = 0
|
||||
def distance2 = 0
|
||||
if(primarySchools && dist.lng){
|
||||
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
|
||||
if(info1 && info1.school_lat2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
|
||||
if(distance1>3000){
|
||||
info1 = null
|
||||
}
|
||||
}else{
|
||||
info1 = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(midSchools) {
|
||||
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
|
||||
if(info2 && info2.school_lng2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
|
||||
if(distance2>3000){
|
||||
info2 = null
|
||||
}
|
||||
}else{
|
||||
info2 = null
|
||||
}
|
||||
}
|
||||
if(info1|| info2){
|
||||
def row = []
|
||||
row << cityCodeMap.get(dist.city)
|
||||
row << regionCodeMap.get(dist.city +"-"+dist.region)
|
||||
row << dist.name
|
||||
row << dist.lng/10000000000.0
|
||||
row << dist.lat/10000000000.0
|
||||
row << dist.address
|
||||
if(info1){
|
||||
row << info1.school_name
|
||||
row << info1.school_address!=null?info1.school_address:''
|
||||
row << info1.attribute!=null?info1.attribute:''
|
||||
row << info1.school_lng2!=null?info1.school_lng2:''
|
||||
row << info1.school_lat2!=null?info1.school_lat2:''
|
||||
// row << distance1
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
|
||||
if(info2){
|
||||
row << info2.school_name
|
||||
row << info2.school_address!=null?info2.school_address:''
|
||||
row << info2.attribute!=null?info2.attribute:''
|
||||
row << info2.school_lng2!=null?info2.school_lng2:''
|
||||
row << info2.school_lat2!=null?info2.school_lat2:''
|
||||
// row << distance2
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
row << ""
|
||||
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << "fjw"
|
||||
row << dist._id.toString()
|
||||
row << "对口"
|
||||
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def selectSchool(lng, lat, def schools){
|
||||
def minSchool = null
|
||||
def level0 = []//国家
|
||||
def level1 = []//省重点
|
||||
def level2 = []//市重点
|
||||
def level3 = []//区重点
|
||||
def level4 = []//普通
|
||||
def level5 = []//其它
|
||||
for(def s : schools){
|
||||
if(s.attribute){
|
||||
if(s.attribute.contains("国家重点")){
|
||||
level0 << s
|
||||
}else if(s.attribute.contains("省重点")){
|
||||
level1 << s
|
||||
}else if(s.attribute.contains("市重点")){
|
||||
level2 << s
|
||||
}else if(s.attribute.contains("区重点")){
|
||||
level3 << s
|
||||
}else if(s.attribute.contains("普通")){
|
||||
level4 << s
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(level0){
|
||||
if(level0.size()==1){
|
||||
minSchool = level0.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level0)
|
||||
}
|
||||
}else if(level1){
|
||||
if(level1.size()==1){
|
||||
minSchool = level1.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level1)
|
||||
}
|
||||
}else if(level2){
|
||||
if(level2.size()==1){
|
||||
minSchool = level2.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level2)
|
||||
}
|
||||
}else if(level3){
|
||||
if(level3.size()==1){
|
||||
minSchool = level3.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level3)
|
||||
}
|
||||
}else if(level4){
|
||||
if(level4.size()==1){
|
||||
minSchool = level4.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level4)
|
||||
}
|
||||
}else if(level5){
|
||||
if(level5.size()==1){
|
||||
minSchool = level5.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level5)
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
|
||||
if(minSchool){
|
||||
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
for(def d : rets){
|
||||
if(!minSchool.school_address){
|
||||
minSchool.school_address = d.address
|
||||
}
|
||||
if(!minSchool.attribute){
|
||||
minSchool.attribute = d.attribute
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(minSchool){
|
||||
if(minSchool.school_map_lng){
|
||||
minSchool.school_lng2 = minSchool.school_map_lng
|
||||
minSchool.school_lat2 = minSchool.school_map_lat
|
||||
}else if(minSchool.school_lng1){
|
||||
minSchool.school_lng2 = minSchool.school_lng1
|
||||
minSchool.school_lat2 = minSchool.school_lat1
|
||||
}else if(minSchool.school_lng){
|
||||
minSchool.school_lng2 = minSchool.school_lng
|
||||
minSchool.school_lat2 = minSchool.school_lat
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return minSchool;
|
||||
}
|
||||
|
||||
|
||||
def findSchoolInfo(def city, def region, def schoolName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", schoolName)
|
||||
def dbcursor = school_col.find(q)
|
||||
def rets = []
|
||||
for(def obj : dbcursor){
|
||||
rets << obj
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def selectMinDistanceSchool(def lat, def lng, def schools){
|
||||
def minSchool = null;
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def s1 : schools){
|
||||
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
if(s1.school_map_lng){
|
||||
lng2 = s1.school_map_lng
|
||||
lat2 = s1.school_map_lat
|
||||
}else if(s1.school_lng1){
|
||||
lng2 = s1.school_lng1
|
||||
lat2 = s1.school_lat1
|
||||
}else if(s1.school_lng){
|
||||
lng2 = s1.school_lng
|
||||
lat2 = s1.school_lat
|
||||
}
|
||||
if(lat2){
|
||||
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
|
||||
println s1.district_name +" "+ s1.school_name+" "+distance
|
||||
if(distance<minDistance){
|
||||
minDistance = distance
|
||||
minSchool = s1
|
||||
minSchool.school_lng2 = lng2
|
||||
minSchool.school_lat2 = lat2
|
||||
}
|
||||
}
|
||||
}
|
||||
return minSchool
|
||||
}
|
||||
|
||||
def findPrimarySchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def findMiddleSchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getRegionCodeByCity(def city){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
}
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
class FZloationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new FZloationUpdate();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
|
||||
def index = 0
|
||||
def locationMap = getMoreSameLocationDist(districts)
|
||||
def total = 0
|
||||
locationMap.each {k,v->
|
||||
total = updateLocation(v, total);
|
||||
println "---------------------------------------------------------------------------------------------------------------"
|
||||
}
|
||||
}
|
||||
|
||||
def updateLocation(def lists, def total){
|
||||
|
||||
|
||||
for(def dist : lists){
|
||||
|
||||
def addrDistMap = getLocation(dist, dist.address)
|
||||
def nameDistMap = getLocation(dist, dist.region + dist.name)
|
||||
|
||||
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
|
||||
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
|
||||
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
|
||||
|
||||
if(addrDistMap){
|
||||
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
|
||||
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
|
||||
|
||||
def baidu = null
|
||||
def lat = null
|
||||
def lng = null
|
||||
def upldateFlag = ""
|
||||
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
|
||||
lat = addrDistMap.lat
|
||||
lng = addrDistMap.lng
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Address"
|
||||
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
|
||||
lng = nameDistMap.lng
|
||||
lat = nameDistMap.lat
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Name"
|
||||
}
|
||||
if(baidu){
|
||||
total++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
Long baiduLat = baidu.y*10000000000L
|
||||
Long baiduLng = baidu.x*10000000000L
|
||||
udata.put("lat2", baiduLat)
|
||||
udata.put("lng2", baiduLng)
|
||||
Long lat0 = lat*10000000000L
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
def getMoreSameLocationDist(def districts){
|
||||
def locationMap = [:]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def locationKey = lng + "-" + lat
|
||||
def list = locationMap.get(locationKey)
|
||||
if(!list){
|
||||
list = []
|
||||
locationMap.put(locationKey, list);
|
||||
}
|
||||
list << dist
|
||||
}
|
||||
def resMap = [:]
|
||||
locationMap.each{k,v->
|
||||
if(v.size()>=2){
|
||||
|
||||
v.sort{ a,b->
|
||||
return a.name.size().compareTo(b.name.size())
|
||||
}
|
||||
resMap.put(k,v)
|
||||
}
|
||||
}
|
||||
|
||||
def lastRetMaps = [:]
|
||||
resMap.each{k,v->
|
||||
def swname = null
|
||||
def count = 0
|
||||
for(def i=0;i<v.size();i++){
|
||||
def d = v[i]
|
||||
if(i==0){
|
||||
if(d.name.contains("期")||d.name.contains("区")){
|
||||
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
|
||||
}else{
|
||||
swname = d.name
|
||||
}
|
||||
}else{
|
||||
if(d.name.contains(swname)||swname.contains(d.name)){
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(count+1!=v.size()){
|
||||
lastRetMaps.put(k,v)
|
||||
}
|
||||
}
|
||||
return lastRetMaps;
|
||||
}
|
||||
|
||||
|
||||
def deleteRegion(def s, def region){
|
||||
def key = ""
|
||||
if(s.indexOf(region)>-1){
|
||||
if(s.indexOf(region+"区")>-1){
|
||||
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"县")>-1){
|
||||
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"市")>-1){
|
||||
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
|
||||
}
|
||||
}else{
|
||||
key = s;
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getLocation(dist, addr){
|
||||
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
if(location){
|
||||
def locMap = location.split(",")
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
|
||||
rMap.distance = distance
|
||||
rMap.level = level
|
||||
rMap.lat = locMap[1].toDouble()
|
||||
rMap.lng = locMap[0].toDouble()
|
||||
rMap.keyword = map.formatted_address
|
||||
rMap.location = location
|
||||
rMap.region = map.district
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
class FzLocationBatch {
|
||||
}
|
||||
+20
-1
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
|
||||
// task.matchHouse();
|
||||
|
||||
// task.villa()
|
||||
task.addressParse()
|
||||
// task.addressParse()
|
||||
|
||||
task.deleteProcess()
|
||||
}
|
||||
|
||||
def repeat(){
|
||||
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
|
||||
return floor;
|
||||
}
|
||||
|
||||
def deleteProcess(){
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
|
||||
def addrList = []
|
||||
for(def d : datas){
|
||||
def addr = d['G']
|
||||
addrList << addr
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
|
||||
def addr = row['C']
|
||||
if(addrList.contains(addr)){
|
||||
row['R'] = "已调查"
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
println "wwww"
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202101);
|
||||
def count2 = findTotalCount(source, 202102);
|
||||
def count1 = findTotalCount(source, 202102);
|
||||
def count2 = findTotalCount(source, 202103);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class AddFangSchoolRegion {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddFangSchoolRegion();
|
||||
//task.process()
|
||||
//task.match()
|
||||
task.addMapLocation()
|
||||
}
|
||||
|
||||
def addMapLocation(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
def data = findSchool(d.city, d.region, d.school_name)
|
||||
if(data && data.location){
|
||||
def location = data.location
|
||||
if(location){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("school_map_lng", location[0])
|
||||
udata.put("school_map_lat", location[1])
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
index ++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def match(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// if(!data.match_name){
|
||||
println i +" "+data.district_name
|
||||
matching(data);
|
||||
// }
|
||||
|
||||
} catch (Exception e) {
|
||||
println "Òì³£"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matching(def it){
|
||||
def rMap = matchDistrict(it.city, it.region, it.district_name)
|
||||
if(!rMap){
|
||||
rMap = matchDistrict(it.city, it.region, it.address)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("match_city", rMap.city)
|
||||
data.put("match_region", rMap.region)
|
||||
data.put("match_name", rMap.name)
|
||||
data.put("match_address", rMap.address)
|
||||
data.put("match_lat", rMap.lat)
|
||||
data.put("match_lng", rMap.lng)
|
||||
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict(def city, def region, def address){
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dto = doc.result
|
||||
if(dto && dto.searchAddress){
|
||||
def ret = dto.matchResult
|
||||
if(ret.match){
|
||||
rMap.city = dto.city
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.lat = dto.lat
|
||||
rMap.lng = dto.lng
|
||||
rMap.id = dto.id
|
||||
rMap.address = dto.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def process(){
|
||||
def datas = findNoRegionData();
|
||||
findSimialData(datas);
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findNoRegionData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findSimialData(datas){
|
||||
for(def d : datas){
|
||||
def obj = findThirdPriceData(d.city, d.url+"/")
|
||||
if(!obj){
|
||||
obj = findThirdPriceByName(d.city, d.district_name)
|
||||
}
|
||||
if(obj){
|
||||
d.region = obj.region
|
||||
if(!d.district_addrss){
|
||||
d.district_address = obj.address
|
||||
}
|
||||
updateDistrictSchool(d);
|
||||
|
||||
}else{
|
||||
println d.school_name+"" + d.district_name+"û²éµ½"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findThirdPriceData(def city, def url){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("district_id", url)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findThirdPriceByName(def city, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("name", name)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def updateDistrictSchool(d){
|
||||
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("region", d.region)
|
||||
data.put("district_address", d.district_address)
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
|
||||
def findSchool(city, region, school_name){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("city", city)
|
||||
data.put("region", region)
|
||||
data.put("name", school_name)
|
||||
return school_col.findOne(data)
|
||||
}
|
||||
}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.alibaba.fastjson.annotation.JSONField
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.NameValuePair
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.http.HttpEntity
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.impl.client.CloseableHttpClient
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
|
||||
import java.nio.charset.Charset
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GushuwangCrawl {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
static main(args) {
|
||||
def task = new GushuwangCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
// chaxuequCrawl()
|
||||
suzhouCrawl();
|
||||
|
||||
}
|
||||
|
||||
def suzhouCrawl(){
|
||||
def page = 1
|
||||
def stop = false
|
||||
def list = []
|
||||
while(!stop){
|
||||
def s = schoolList(page);
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.ok){
|
||||
def datas = jsonObject.extra.list
|
||||
if(datas){
|
||||
list.addAll(datas)
|
||||
page++
|
||||
println page
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(list){
|
||||
def schools = []
|
||||
for(def d : list){
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = "苏州"
|
||||
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(d.schoolname)
|
||||
schoolData.name = d.schoolname
|
||||
schoolData.address = d.adr
|
||||
/**来源网站*/
|
||||
schoolData.source = "苏州学区查询"
|
||||
/**房源详情url*/
|
||||
schools << schoolData
|
||||
}
|
||||
|
||||
saveChaxuequSchool(schools)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def chaxuequCrawl(){
|
||||
|
||||
def page = 1
|
||||
def stop = false
|
||||
|
||||
def schools = []
|
||||
|
||||
while(!stop){
|
||||
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
|
||||
println "内容:"+s
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject.code == 200){
|
||||
def data = jsonObject.data
|
||||
def totalPage = data.pages
|
||||
if(data.list){
|
||||
page++
|
||||
if(page+1>totalPage){
|
||||
stop = true;
|
||||
}
|
||||
schools.addAll(data.list)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
print(schools.size());
|
||||
|
||||
def districtSchools = []
|
||||
def schoolInfos = []
|
||||
if(schools.size()>0){
|
||||
|
||||
def url = "https://51yongka.com/fapi/school/details"
|
||||
for(def d : schools){
|
||||
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
|
||||
def s = chaxuequInfoCrawl(url, params)
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.code==200){
|
||||
|
||||
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
|
||||
def districts = jsonObject.data.nearbyPropertyList
|
||||
for(def dist : districts){
|
||||
def dataMap = new BasicDBObject()
|
||||
dataMap.city = jsonObject.data.city
|
||||
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
dataMap.category = converCategory(jsonObject.data.schoolName)
|
||||
dataMap.district_name = dist.name.trim()
|
||||
dataMap.district_address = dist.address
|
||||
dataMap.school_name = jsonObject.data.schoolName.trim()
|
||||
dataMap.school_address = jsonObject.data.address
|
||||
dataMap.source = "查学区房"
|
||||
dataMap.url = url + "?"+params
|
||||
dataMap.telephone = jsonObject.data.officialTelephone;
|
||||
districtSchools << dataMap
|
||||
}
|
||||
}else{
|
||||
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = jsonObject.data.city
|
||||
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(jsonObject.data.schoolName)
|
||||
schoolData.name = jsonObject.data.schoolName.trim()
|
||||
schoolData.address = jsonObject.data.address
|
||||
/**来源网站*/
|
||||
schoolData.source = "查学区房"
|
||||
/**房源详情url*/
|
||||
schoolData.telephone = jsonObject.data.officialTelephone;
|
||||
schoolInfos << schoolData
|
||||
println jsonObject.data.schoolName +" 无周边楼盘"
|
||||
}
|
||||
|
||||
}
|
||||
println s;
|
||||
}
|
||||
}
|
||||
|
||||
println "小区:"+districtSchools.size()
|
||||
// saveChaxuequDistrictSchool(districtSchools)
|
||||
saveChaxuequSchool(schoolInfos)
|
||||
|
||||
}
|
||||
|
||||
def saveChaxuequSchool(schoolInfos){
|
||||
for(def ds : schoolInfos){
|
||||
def obj = findSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveChaxuequDistrictSchool(def districtSchools){
|
||||
for(def ds : districtSchools){
|
||||
def obj = findDistrictSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
district_school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findSchoolData(city, school_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
return school_col.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
def findDistrictSchoolData(city, school_name, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
q.put("district_name", district_name)
|
||||
return district_school_col.findOne(q)
|
||||
}
|
||||
|
||||
def converCategory(def schoolName){
|
||||
|
||||
if(schoolName){
|
||||
if(schoolName.contains("初中部")||schoolName.contains("中学")){
|
||||
return "初中"
|
||||
}else if(schoolName.contains("小学")){
|
||||
return "小学"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def static chaxuequInfoCrawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","51yongka.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","75")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
|
||||
headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def static schoolList(page) {
|
||||
def responseMap = [:]
|
||||
String s = null
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","m.gusuwang.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","40")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
|
||||
//params信息
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
|
||||
// //post请求
|
||||
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
|
||||
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
outputStream.write(content.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
// byte[] data = new byte[1024];
|
||||
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
|
||||
// while (inputStream.read(data) != -1) {
|
||||
// String s = new String(data, Charset.forName("utf-8"));
|
||||
// sb.append(s);
|
||||
// }
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -52,7 +52,7 @@ class ElasticUtils {
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
|
||||
+3
-3
@@ -7,14 +7,14 @@ import java.util.regex.Pattern;
|
||||
*/
|
||||
public interface AddressParserPatterns {
|
||||
|
||||
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
|
||||
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
|
||||
Pattern STREET = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?");
|
||||
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零|一三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
|
||||
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
|
||||
Pattern STAGE_PATTERN = Pattern.compile("(([零一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([零一二三四五六七八九十百千\\d]+)期");
|
||||
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|零|一|二|三|四|五|六|七|八|九|十)+(区|坊)");
|
||||
Pattern TOWN = Pattern.compile("^.+?(乡|镇|街道)+");
|
||||
Pattern NUMBER = Pattern.compile("\\d+");
|
||||
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
|
||||
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
|
||||
Pattern DISTRICT = Pattern.compile("^.{2,}?(区|县)");
|
||||
|
||||
}
|
||||
|
||||
+80
-19
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.service.address;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.CNNMFilter;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.GeoUtils;
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
@@ -17,6 +18,7 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors;
|
||||
import com.fangjia.service.address.processor.AddressProcessors;
|
||||
import com.fangjia.service.api.AliyunService;
|
||||
import com.fangjia.service.bo.AliyunBo;
|
||||
import com.fangjia.service.bo.MatchAddressScore;
|
||||
import com.fangjia.service.datastore.DistrictService;
|
||||
import com.fangjia.service.mongo.addrepo.AddressRepoService;
|
||||
import com.fangjia.service.mongo.bkoff.HouseLoudongService;
|
||||
@@ -170,7 +172,9 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
if(aliyunBo != null){
|
||||
String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", "");
|
||||
if(StringUtils.isNotBlank(poi)){
|
||||
addr.setKeywords(poi);
|
||||
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
|
||||
addr.setKeywords(poi);
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getCity())){
|
||||
addr.setSuggestCity(aliyunBo.getCity());
|
||||
@@ -189,6 +193,45 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
if(params.isUseAliyunApi() && aliyunBo != null){
|
||||
completBuildingField(matchedBuilding, aliyunBo, params);
|
||||
}
|
||||
Address tempAddr = matchedBuilding.getSearchAddress();
|
||||
if(StringUtils.isBlank(tempAddr.getProvince()) || StringUtils.isBlank(tempAddr.getCity()) || StringUtils.isBlank(tempAddr.getDistrict())|| StringUtils.isBlank(tempAddr.getStreet())){
|
||||
// List<AddressMatchMetaData> metaDatas = addressRepoService.findAllByKeyword(null, params.getBuildingType(), new ArrayList<String>() {{add(tempAddr.getOrgAddress());}}, tempAddr.getLng(), tempAddr.getLat(), 100, false);
|
||||
tempAddr.addKeyword(tempAddr.getOrgAddress());
|
||||
List<AddressMatchMetaData> metaDatas = findMatchMetaData(tempAddr, params.getBuildingType(), MetaDataType.MAP, false);
|
||||
// String addressStr = (StringUtils.isBlank(tempAddr.getStreet())?"":tempAddr.getStreet()) + (StringUtils.isBlank(tempAddr.getStreetNumber())?"":tempAddr.getStreetNumber());
|
||||
Map<String,List<AddressMatchMetaData>> metaDataMap = new HashMap<String, List<AddressMatchMetaData>>();
|
||||
if(StringUtils.isBlank(tempAddr.getDistrict()) && !metaDatas.isEmpty()){
|
||||
for(AddressMatchMetaData metaData:metaDatas){
|
||||
if(StringUtils.isNotBlank(metaData.getMapCity())){
|
||||
List<AddressMatchMetaData> metaList = metaDataMap.containsKey(metaData.getMapCity()) ? metaDataMap.get(metaData.getMapCity()) : new ArrayList<AddressMatchMetaData>();
|
||||
metaList.add(metaData);
|
||||
metaDataMap.put(metaData.getMapCity(),metaList);
|
||||
}
|
||||
}
|
||||
String tempCity = "";
|
||||
if(StringUtils.isNotBlank(tempAddr.getCity()) && metaDataMap.containsKey(tempAddr.getCity())){
|
||||
tempCity = tempAddr.getCity();
|
||||
}else{
|
||||
int size = 0;
|
||||
for(String key:metaDataMap.keySet()){
|
||||
if(metaDataMap.get(key).size() > size){
|
||||
size = metaDataMap.get(key).size();
|
||||
tempCity = key;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(tempCity)){
|
||||
AddressMatchMetaData meta = metaDataMap.get(tempCity).get(0);
|
||||
tempAddr.setProvince(meta.getMapProvince());
|
||||
tempAddr.setCity(meta.getMapCity());
|
||||
tempAddr.setDistrict(meta.getDistrict());
|
||||
if((tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){
|
||||
matchedBuilding.setAddress(meta.getAddress());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
matchedBuilding.setSearchAddress(tempAddr);
|
||||
return matchedBuilding;
|
||||
}
|
||||
|
||||
@@ -224,12 +267,25 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
if(StringUtils.isBlank(addr.getTownship())){
|
||||
addr.setTownship(aliyunBo.getTown());
|
||||
}
|
||||
addr.setOrgBuildingNumber(aliyunBo.getHouseno());
|
||||
/*addr.setOrgBuildingNumber(aliyunBo.getHouseno());
|
||||
addr.setBuildingNumber(aliyunBo.getHouseno());
|
||||
addr.setOrgUnitNumber(aliyunBo.getCellno());
|
||||
addr.setUnitNumber(aliyunBo.getCellno());
|
||||
addr.setOrgRoomNumber(aliyunBo.getRoomno());
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
addr.setUnitNumber(aliyunBo.getCellno());*/
|
||||
if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(²ã|Â¥)$")){
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
}else{
|
||||
addr.setRoomNumber(null);
|
||||
addr.setRoomNumber(null);
|
||||
}
|
||||
|
||||
/*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", "");
|
||||
if(orgRoom!=null && orgRoom.equals(aliRoom)){
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
}else{
|
||||
addr.setRoomNumber(null);
|
||||
}*/
|
||||
|
||||
if(StringUtils.isBlank(addr.getStreet())){
|
||||
addr.setStreet(aliyunBo.getRoad());
|
||||
@@ -362,7 +418,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
* @param metaDataType
|
||||
* @return
|
||||
*/
|
||||
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType){
|
||||
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType, boolean validCity){
|
||||
if(MetaDataType.MAP == metaDataType){
|
||||
List<String> keywords = address.getKeywordList();
|
||||
if(keywords.isEmpty()){
|
||||
@@ -375,7 +431,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
}
|
||||
keywords.add(standardAddress);
|
||||
}
|
||||
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100);
|
||||
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100, validCity);
|
||||
}else{
|
||||
return SEAWEED_LOCAL_CACHE.get(new MetaDataCacheKey(address.getCity(), buildingType, MetaDataType.CPDB));
|
||||
}
|
||||
@@ -517,20 +573,20 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
scoreResult.setOrgAddress(addr.getOrgAddress());
|
||||
if(buildingType != null) {
|
||||
if(MetaDataType.MAP == metaDataType){
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP);
|
||||
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB));
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP, true);
|
||||
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true));
|
||||
}else{
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB);
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true);
|
||||
}
|
||||
} else {
|
||||
if(MetaDataType.MAP == metaDataType){
|
||||
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP);
|
||||
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP, true);
|
||||
}
|
||||
for (BuildingType bt : addr.guessBuildType()) {
|
||||
if(metaDatas == null){
|
||||
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB);
|
||||
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB, true);
|
||||
}else{
|
||||
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB));
|
||||
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -967,7 +1023,12 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
formattedAddressBuilder.append(address.getStreetNumber());
|
||||
}
|
||||
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
|
||||
formattedAddressBuilder.append(building.getAddress());
|
||||
Address orgAddr = new Address();
|
||||
orgAddr.setOrgAddress(building.getAddress());
|
||||
Address addr = AddressProcessors.process(orgAddr);
|
||||
building.getSearchAddress().setStreet(addr.getStreet());
|
||||
building.getSearchAddress().setStreetNumber(addr.getStreetNumber());
|
||||
formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(building.getSuggestName())){
|
||||
formattedAddressBuilder.append(building.getSuggestName());
|
||||
@@ -985,16 +1046,16 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
&& address.getStreetNumber().endsWith(address.getOrgBuildingNumber())){
|
||||
|
||||
}else{
|
||||
formattedAddressBuilder.append(address.getBuildingNumber());
|
||||
// formattedAddressBuilder.append(address.getBuildingNumber() + "¶°");
|
||||
// formattedAddressBuilder.append(address.getBuildingNumber());
|
||||
formattedAddressBuilder.append(address.getBuildingNumber() + "¶°");
|
||||
}
|
||||
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getOrgUnitNumber())){
|
||||
formattedAddressBuilder.append(address.getUnitNumber());
|
||||
// formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª");
|
||||
// formattedAddressBuilder.append(address.getUnitNumber());
|
||||
formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª");
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getOrgRoomNumber())){
|
||||
if(StringUtils.isNotBlank(address.getRoomNumber())){
|
||||
formattedAddressBuilder.append(address.getRoomNumber());
|
||||
// formattedAddressBuilder.append(address.getRoomNumber() + "ÊÒ");
|
||||
}
|
||||
|
||||
+32
-1
@@ -259,7 +259,38 @@ public class AddressFormatterFactory {
|
||||
@Override
|
||||
protected String doFormat(String address) {
|
||||
String matchStr = DataUtil.match(address, TARGET);
|
||||
String formatStr = matchStr + "ºÅ";
|
||||
String formatStr = matchStr.replaceAll("段-","段");
|
||||
if(!matchStr.endsWith("号")){
|
||||
formatStr += "号";
|
||||
}
|
||||
return address.replace(matchStr,formatStr);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 剑南路东段-12号-18号
|
||||
*/
|
||||
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))")) {
|
||||
final Pattern TARGET = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))");
|
||||
@Override
|
||||
protected String doFormat(String address) {
|
||||
String matchStr = DataUtil.match(address, TARGET);
|
||||
String formatStr = matchStr;
|
||||
if(!matchStr.endsWith("号")){
|
||||
formatStr += "号";
|
||||
}
|
||||
return address.replace(matchStr,formatStr);
|
||||
}
|
||||
});
|
||||
/**
|
||||
* 剑南路东段-12号-18号
|
||||
*/
|
||||
formatters.add(new AbstractAddressFormatter(Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))")) {
|
||||
final Pattern TARGET = Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))");
|
||||
@Override
|
||||
protected String doFormat(String address) {
|
||||
String matchStr = DataUtil.match(address, TARGET);
|
||||
String formatStr = matchStr.replace("-","");
|
||||
return address.replace(matchStr,formatStr);
|
||||
}
|
||||
});
|
||||
|
||||
+18
@@ -44,6 +44,24 @@ public class AddressMatchMetaData {
|
||||
private Double lat;
|
||||
private String map;
|
||||
|
||||
public String getMapProvince() {
|
||||
return mapProvince;
|
||||
}
|
||||
|
||||
public void setMapProvince(String mapProvince) {
|
||||
this.mapProvince = mapProvince;
|
||||
}
|
||||
|
||||
public String getMapCity() {
|
||||
return mapCity;
|
||||
}
|
||||
|
||||
public void setMapCity(String mapCity) {
|
||||
this.mapCity = mapCity;
|
||||
}
|
||||
|
||||
private String mapProvince;
|
||||
private String mapCity;
|
||||
public void addName(String name, float priority){
|
||||
if(StringUtils.isNotBlank(name)){
|
||||
String formatName = AddressParserUtil.standardAddressAndName(name);
|
||||
|
||||
+15
-6
@@ -50,18 +50,23 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(city)){
|
||||
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
String possibleCity = null;
|
||||
String possibleCity = parseCityFromAddress(orgAddress, address);
|
||||
String confirmCity = null;
|
||||
/**如果省份和城市信息相互矛盾 则用区域推断城市*/
|
||||
if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){
|
||||
List<String> possibleCities = speculateCityNameByRegion(address, city);
|
||||
if(isUnique(possibleCities)){
|
||||
formatCity = getFirst(possibleCities);
|
||||
confirmCity = formatCity;
|
||||
}
|
||||
}
|
||||
if(null != formatCity){
|
||||
city = formatCity;
|
||||
if(formatCity == null || !formatCity.equals(possibleCity)){
|
||||
confirmCity = possibleCity;
|
||||
}
|
||||
if(null != confirmCity){
|
||||
city = confirmCity;
|
||||
address.setCity(city);
|
||||
address.setCityStatus(CityStatus.SURE);
|
||||
}
|
||||
@@ -98,7 +103,7 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
address.setCity(city);
|
||||
}
|
||||
if(StringUtils.isNotBlank(city) && scoreRegion(city, orgAddress) > 0.4){
|
||||
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?" , "");
|
||||
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?(?!(路|街|大道|巷|道|段))" , "");
|
||||
address.setAddress(orgAddress);
|
||||
address.setCityStatus(CityStatus.SURE);
|
||||
return RepeatStatus.NEXT;
|
||||
@@ -166,7 +171,7 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
String street = getStreet(address);
|
||||
for(String city : allCity){
|
||||
if(street != null){
|
||||
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)"), street);
|
||||
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
|
||||
if(matchResult !=null){
|
||||
address = address.replace(matchResult.matchContent, "");
|
||||
}
|
||||
@@ -199,6 +204,10 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
}
|
||||
Map<String, List<String>> regionMap = RegionData.getAllRegions(city);
|
||||
for(String region : regionMap.keySet()){
|
||||
MatchResult matchResult = matchFirst(Pattern.compile(region + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
|
||||
if(matchResult !=null){
|
||||
address = address.replace(matchResult.matchContent, "");
|
||||
}
|
||||
if(address.indexOf(region, index) > -1){
|
||||
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
|
||||
weight.score += 2;
|
||||
|
||||
+2
-2
@@ -24,9 +24,9 @@ public class ProvinceProcessor extends AbstractAddressProcessor {
|
||||
|
||||
String orgAddress = address.getAddress();
|
||||
String[] provinceInfo = getProvince(orgAddress);
|
||||
if(provinceInfo == null && address.getCity() != null){
|
||||
/*if(provinceInfo == null && address.getCity() != null){
|
||||
provinceInfo = getProvince(address.getCity());
|
||||
}
|
||||
}*/
|
||||
if (null != provinceInfo) {
|
||||
String province = provinceInfo[NAME_IDX];
|
||||
if (provinceInfo[FULL_NAME_IDX] != null) {//全称匹配
|
||||
|
||||
+12
-4
@@ -18,6 +18,7 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
|
||||
static final Pattern NUMBER_END = Pattern.compile("(?<=\\D)\\d{3,4}$");
|
||||
static final Pattern HOUSE_NUMBER = Pattern.compile("\\d{3,4}");
|
||||
static final String HOUSE_UNITS_PATTERN = "(ÊÒ|·¿|ºÅ·¿|ºÅ(?!Ôº)|»§|ÆÌ)+";
|
||||
static final Pattern HOUSE_PATTERN_CASE02 = Pattern.compile("(\\d+(ÊÒ|·¿|ºÅ)?[¡¢]?)+$");
|
||||
|
||||
@Override
|
||||
public RepeatStatus process(Address address) {
|
||||
@@ -27,10 +28,17 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
|
||||
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
|
||||
address.setOrgRoomNumber(matchResult.matchContent);
|
||||
address.setAddress(orgAddress);
|
||||
analyFloorUseRoomNumber(address);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
MatchResult matchResult2 = matchFirst(HOUSE_PATTERN_CASE02, orgAddress);
|
||||
if (matchResult2 != null) {
|
||||
orgAddress = removeSubstring(orgAddress, matchResult2.start, matchResult2.end);
|
||||
address.setOrgRoomNumber(matchResult2.matchContent);
|
||||
address.setAddress(orgAddress);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
|
||||
matchResult = matchFirst(HOUSE_PATTERN, orgAddress);
|
||||
if (matchResult != null) {
|
||||
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
|
||||
@@ -39,10 +47,10 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
|
||||
} else {
|
||||
address.setOrgRoomNumber(matchResult.matchContent + "ÊÒ");
|
||||
}
|
||||
analyFloorUseRoomNumber(address);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
address.setAddress(orgAddress);
|
||||
}
|
||||
analyFloorUseRoomNumber(address);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
removeHouseUnit(address);
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
|
||||
+1
@@ -41,6 +41,7 @@ public class TooltipProcessor extends AbstractAddressProcessor {
|
||||
offset += matchResult.matchContent.length();
|
||||
if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){
|
||||
address.setTooltip(tooltip);
|
||||
address.setStreet(getStreet(tooltip));
|
||||
} else if(containsDistrictName(tooltip)){
|
||||
address.setTooltip(tooltip);
|
||||
} else if(RegionData.isRegion(address.getCity(), tooltip)){
|
||||
|
||||
+10
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.service.address.processor;
|
||||
|
||||
import com.fangjia.service.address.mode.Address;
|
||||
import com.fangjia.service.cnAnalyzer.Token;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -14,6 +15,15 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
|
||||
@Override
|
||||
public AbstractAddressProcessor.RepeatStatus process(Address address) {
|
||||
String orgAddress = address.getAddress();
|
||||
if(address.getTokens()!=null){
|
||||
for(Token token:address.getTokens()){
|
||||
if("district".equals(token.getKeywordTypes())){
|
||||
address.setKeywords(token.getName());
|
||||
address.setAddress(orgAddress.replace(token.getName(),""));
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
}
|
||||
}
|
||||
List<String> names = tokenDistrictNames(orgAddress);
|
||||
List<String> kws = new ArrayList<String>();
|
||||
for(String name : names){
|
||||
|
||||
@@ -68,7 +68,7 @@ public interface AmapPoiService {
|
||||
* @param types
|
||||
* @return
|
||||
*/
|
||||
List<AmapPoi> search(String city, String keywords, Integer limit, String types);
|
||||
List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity);
|
||||
|
||||
|
||||
/**
|
||||
|
||||
+12
-9
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.service.api;
|
||||
|
||||
import com.fangjia.core.ProvinceData;
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.GeoUtils;
|
||||
@@ -258,14 +259,14 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AmapPoi> search(String city, String keywords, Integer limit, String types) {
|
||||
public List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
String area = city;
|
||||
params.put("city", area);
|
||||
params.put("output", "json");
|
||||
params.put("keywords", keywords);
|
||||
params.put("key", "4ccfe3c898a9489884bd2bcb1ef6dcde");
|
||||
params.put("citylimit", "true");
|
||||
// params.put("citylimit", "true");
|
||||
params.put("offset", limit == null?"25":String.valueOf(limit));
|
||||
if(types != null){
|
||||
params.put("types", "120301&120302&120201&060101");
|
||||
@@ -351,15 +352,17 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
cityname = poi.getString("cityname");
|
||||
}
|
||||
cityname = RegionData.formatCity(cityname);
|
||||
if(!city.equals(cityname)){
|
||||
if((city!=null && !city.equals(cityname) && validCity) || cityname == null){
|
||||
continue;
|
||||
}
|
||||
pname = ProvinceData.formatProvince(pname);
|
||||
AmapPoi amapPoi = new AmapPoi();
|
||||
amapPoi.setCity(city);
|
||||
amapPoi.setCity(cityname);
|
||||
amapPoi.setPname(pname);
|
||||
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
|
||||
adname = poi.getString("adname");
|
||||
String region = RegionData.formatRegion(city, adname);
|
||||
if(RegionData.isRegion(city, region)){
|
||||
String region = RegionData.formatRegion(cityname, adname);
|
||||
if(RegionData.isRegion(cityname, region)){
|
||||
amapPoi.setRegion(region);
|
||||
}
|
||||
}
|
||||
@@ -380,8 +383,8 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
amapPoi.setAlias(alias);
|
||||
}
|
||||
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
|
||||
String region = RegionData.formatRegion(city, poi.getString("adname"));
|
||||
if(RegionData.isRegion(city, region)){
|
||||
String region = RegionData.formatRegion(cityname, poi.getString("adname"));
|
||||
if(RegionData.isRegion(cityname, region)){
|
||||
amapPoi.setRegion(region);
|
||||
}
|
||||
}
|
||||
@@ -391,7 +394,7 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
.replace(cityname, "").replace(adname, "");
|
||||
amapPoi.setAddress(AddressParserUtil.optimizeAddress(address));
|
||||
}
|
||||
amapPoi.setCity(city);
|
||||
amapPoi.setCity(cityname);
|
||||
if(poi.containsKey("location") && poi.get("location") instanceof String){
|
||||
String[] location = poi.getString("location").split(",");
|
||||
amapPoi.setLng(Double.valueOf(location[0]));
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ public interface AddressRepoService {
|
||||
* @param max
|
||||
* @return
|
||||
*/
|
||||
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max);
|
||||
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity);
|
||||
|
||||
/**
|
||||
*
|
||||
|
||||
+8
-2
@@ -87,7 +87,7 @@ public class AddressRepoServiceImpl implements AddressRepoService {
|
||||
return addrs;
|
||||
}
|
||||
|
||||
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max){
|
||||
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity){
|
||||
String types = null;
|
||||
String tag = null;
|
||||
String category = null;
|
||||
@@ -104,7 +104,11 @@ public class AddressRepoServiceImpl implements AddressRepoService {
|
||||
tag = "°Ù»õÉ̳¡";
|
||||
category = "×ÛºÏÉ̳¡";
|
||||
}
|
||||
List<AmapPoi> pois = amapPoiService.search(city, StringUtils.join(keywords, "|"), 20, types);
|
||||
List<AmapPoi> pois = new ArrayList<AmapPoi>();
|
||||
for(String keyword:keywords){
|
||||
List<AmapPoi> poiList = amapPoiService.search(city, keyword, 20, types, validCity);
|
||||
pois.addAll(poiList);
|
||||
}
|
||||
List<AddressMatchMetaData> matchMetaDatas = new ArrayList<AddressMatchMetaData>(pois.size());
|
||||
for(AmapPoi poi: pois){
|
||||
AddressMatchMetaData data = new AddressMatchMetaData();
|
||||
@@ -121,6 +125,8 @@ public class AddressRepoServiceImpl implements AddressRepoService {
|
||||
data.setType(poi.getType());
|
||||
data.setMap("¸ßµÂ");
|
||||
data.setCity(city);
|
||||
data.setMapCity(poi.getCity());
|
||||
data.setMapProvince(poi.getPname());
|
||||
if(poi.getAlias() != null){
|
||||
for(String alia : poi.getAlias()){
|
||||
data.addName(alia, MatchText.ALIAS_PRIORITY);
|
||||
|
||||
+6
-1
@@ -31,10 +31,15 @@ public interface SeawaterService {
|
||||
|
||||
void deleteById(String id);
|
||||
|
||||
/**
|
||||
* 批量删除,英文逗号分割,如1,2,3
|
||||
* @param ids
|
||||
*/
|
||||
void deleteByIds(String ids);
|
||||
|
||||
@Deprecated
|
||||
List<Seawater> findAllSeawater(String category, String city, String name, String region, Integer room, String root, String sort, Integer startDate, Integer endDate, int start, int limit, boolean order, String[] fields);
|
||||
|
||||
@Deprecated
|
||||
int findAllSeawaterCount(String category, String city, String name, String region, Integer room, String root, Integer startDate, Integer endDate);
|
||||
|
||||
}
|
||||
|
||||
+7
-4
@@ -12,10 +12,7 @@ import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.solr.client.solrj.util.ClientUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
|
||||
public class SeawaterServiceImpl implements SeawaterService {
|
||||
private SeawaterDao seawaterDao;
|
||||
@@ -188,6 +185,12 @@ public class SeawaterServiceImpl implements SeawaterService {
|
||||
seawaterDao.deleteById(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByIds(String ids) {
|
||||
String[] idArr = ids.split(",");
|
||||
seawaterDao.deleteByIds(Arrays.asList(idArr));
|
||||
}
|
||||
|
||||
public void setSeawaterDao(SeawaterDao seawaterDao) {
|
||||
this.seawaterDao = seawaterDao;
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-core</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>0.2.0</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -154,7 +154,7 @@
|
||||
<dependency>
|
||||
<groupId>com.poseidon</groupId>
|
||||
<artifactId>poseidon-message</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<version>0.0.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.mongodb</groupId>
|
||||
|
||||
Reference in New Issue
Block a user