This commit is contained in:
2021-04-16 19:25:58 +08:00
38 changed files with 2656 additions and 135 deletions
+2 -2
View File
@@ -29,7 +29,7 @@
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-core</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -154,7 +154,7 @@
<dependency>
<groupId>com.poseidon</groupId>
<artifactId>poseidon-message</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.1</version>
<exclusions>
<exclusion>
<groupId>org.mongodb</groupId>
+1 -1
View File
@@ -99,7 +99,7 @@
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-web</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.0</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
@@ -12,6 +12,7 @@ import com.fangjia.service.mongo.fangjia.SeawaterService;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.sothis.core.util.Pager;
import org.sothis.web.mvc.annotation.Param;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
@@ -116,6 +117,14 @@ public class SeawaterController {
return ret;
}
public Object batchDeleteAction(@Param(name="ids") String ids) throws UnsupportedEncodingException {
JsonModel ret = new JsonModel();
if (StringUtils.isNotBlank(ids)){
seawaterService.deleteByIds(ids);
}
return ret;
}
public void setSeawaterService(SeawaterService seawaterService) {
this.seawaterService = seawaterService;
}
@@ -118,9 +118,13 @@
</td>
</tr>
</table>
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
<div>
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
<input class ="btn btn-primary" id="batchDelete" type="submit" value="批量删除" style="float: right" />
</div>
<table id="dataSheet" class="table table-hover table-condensed" name="reginTable">
<tr>
<th style="width:30px;"><input type="checkbox" id="checkAll"/>全选</th>
<th style="width:60px;">类型</th>
<th style="width:50px;">区域</th>
<th style="width:130px;">小区名</th>
@@ -142,6 +146,7 @@
<#if seawaterList??>
<#list seawaterList as it>
<tr>
<td><input type="checkbox" name="houseCheck" value="${it.id}"/></td>
<td>
<#if it.category! == 'sell'>
二手房
@@ -249,7 +254,7 @@
}
});
}
changeCity('${cityDisplay!}');
$(".hrefClick").click(function(){
@@ -336,8 +341,48 @@
$("#roomSpan").hide();
}
}
// 全选
$("#checkAll").click(function(){
$('input[name="houseCheck"]').each(function() {
var _this = this
if (_this.checked == true) {
_this.checked = false
} else {
_this.checked = true
}
})
})
$("#batchDelete").click(function(){
if(confirm('确认要删除所选的数据吗?')) {
var ids = "";
$('input:checkbox[name=houseCheck]:checked').each(function() {
var _this = $(this);
ids += _this.val() + ","
});
if (ids == "") {
alert("请选择数据")
return false;
}
jQuery.ajax({
type: 'POST',
url: '../seawater/batchDelete',
data:{ids:ids},
dataType: 'json',
timeout: 100000,
success: function(model) {
alert("批量删除操作成功");
location.reload();
},
error : function() {
//alert("操作失败");
}
});
}
});
</script>
</html>
</#compress>
@@ -205,7 +205,6 @@ public abstract class ElasticDaoImpl<E extends MongoEntity> extends AbstractJpaC
@Override
public void deleteByIds(List<String> idList) {
}
@Override
@@ -14,10 +14,8 @@ public interface SeawaterDao extends com.fangjia.dal.SeawaterDao<Seawater> {
Seawater findByIdFromEs(String id,String table);
List<Seawater> findSeawatersFromEs(String category, String city, String region, String name, List<String> alias, List<String> regionAlias, Integer startDate, Integer endDate,
Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root, String sort, boolean order, int start, int limit , String table);
List<Seawater> findHouses(String city, String region, List<String> alias, Integer minPrice, Integer maxPrice, Integer limit);
}
@@ -96,6 +96,44 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
return new ArrayList<Seawater>();
}
@Override
public void deleteById(String id) {
deleteByIds(Arrays.asList(id));
}
private Map<String, String> buildHeader() {
Base64 token = new Base64();
String authenticationEncoding = token.encodeAsString(new String("" + ES_USER + ":" + ES_PASSWORD + "").getBytes());
Map<String, String> headers = new HashMap<String, String>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Basic " + authenticationEncoding + "");
return headers;
}
@Override
public void deleteByIds(List<String> idList) {
HttpRequest request = new HttpRequest();
JSONObject reqBody = new JSONObject();
JSONObject query = new JSONObject();
JSONObject bool = new JSONObject();
JSONArray must = new JSONArray();
must.add(addTerms("_id", idList));
bool.put("must", must);
query.put("bool", bool);
reqBody.put("query", query);
JSONObject script = new JSONObject();
script.put("source", "ctx._source.status=1");
reqBody.put("script", script);
try {
request.postWithBody("http://es.fangjia.com:9200/seawater/_update_by_query", reqBody, this.buildHeader(), "UTF-8");
} catch (HttpRequestException e) {
e.printStackTrace();
}
}
private JSONObject addMatch(String name, String value){
JSONObject term = new JSONObject();
JSONObject match = new JSONObject();
@@ -338,6 +376,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
if(limit >= 0){
reqBody.put("size", limit);
}
// ²»Õ¹Ê¾É¾³ý
JSONObject param = new JSONObject();
param.put("status", 0);
JSONObject temp = new JSONObject();
temp.put("term", param);
jsonList.add(temp);
query.put("must", jsonList);
q.put("bool", query);
reqBody.put("query", q);
@@ -1790,6 +1790,7 @@ public class AddressController {
}
}
dto.append("buildingNumber", address.getBuildingNumber());
dto.append("floor", address.getFloor());
dto.append("unitNumber", address.getUnitNumber());
dto.append("roomNumber", address.getRoomNumber());
dto.append("formattedAddress", address.getFormattedAddress());
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "工商大楼");
// q.put("name","东寓新村")
DBObject keys = new BasicDBObject();
if(fields){
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
houseLoudng.fitment_cost = ret.fitment_charge
houseLoudng.car_port = ret.carport
houseLoudng.house_height = ret.house_hight
houseLoudng.worry_facility" : null,
houseLoudng.fitment_age = ret.fitment_age
houseLoudng.lighting_ventilation" : null,
houseLoudng.build_type = ret.build_type
houseLoudng.house_property" : null,
houseLoudng.noise" : null,
houseLoudng.sight" : null,
houseLoudng.fitment_rank" : null,
houseLoudng.sideset" : null,
// houseLoudng.worry_facility" : null,
// houseLoudng.fitment_age = ret.fitment_age
// houseLoudng.lighting_ventilation" : null,
// houseLoudng.build_type = ret.build_type
// houseLoudng.house_property" : null,
// houseLoudng.noise" : null,
// houseLoudng.sight" : null,
// houseLoudng.fitment_rank" : null,
// houseLoudng.sideset" : null,
houseLoudng.district_no = district._id
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,117 @@
package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
class LogStat {
static main(args) {
def task = new LogStat()
task.run()
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\建行\\建行匹配小区2.xlsx", 0, 1)
println datas.size()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行匹配小区2.xlsx",{ d ->
def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
def weekend = DateUtils.getFirstDayOfWeek(calDate)
def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
if(dbo){
d['E'] = dbo.avg_price
}
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(district){
def startDate = DateUtil.addMonth(calDate, -6)
def start = startDate.replace("-","").toInteger()
def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
d['G'] = count
}
},0)
}
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
String sql = "select * from seawater-20* where cat=sell and city='${city}' and region='${region}' and district_name in (${alias.join(',')})"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
rets << data
println data
})
return rets.size()
}
// def findTestDatas(def city, def value) {
// def q = new BasicDBObject()
// q.put("user_id", "1007751")
// q.put("parameter_map.city", city)
// q.put("request_uri", "/property/evaluateForTest")
//
// def keys = new BasicDBObject()
// keys.put("res_map",1)
// def cursor = LOG.find(q)
// def count = 0
// println "总量:"+cursor.count()
// def datas = []
// for(def d : cursor){
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
// def resMap = d.res_map.result
// if(resMap.calcType==value){
// def data = []
// def rowMap = [:]
// rowMap.city = resMap.city
// rowMap.region = resMap.district
// rowMap.name = resMap.name
// rowMap.calDate = d.parameter_map.calDate
// data << rowMap
// }
// }
// }
// return datas;
// }
// for(def i=0;i<datas.size();i++){
// def d = datas[i];
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
//
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(district){
// def startDate = DateUtil.addMonth(calDate, -6)
// def start = startDate.replace("-","").toInteger()
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
// def alias = []
// if(district.alias){
// if(!district.alias.contains(district.name)){
// district.alias << district.name
// }
// alias = district.alias
// }else{
// alias << district.name
// }
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
// d['G'] = count
// }
}
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.address
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class AddressParser {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddressParser();
task.run()
// task.gaoDeAddress()
}
def gaoDeAddress(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match2(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def run(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def match(id, city, region, address, name){
def locationMap = null;
// if(region && !name.contains(region)){
// locationMap = matchLocation(city, region + name, "name")
// }else{
locationMap = matchLocation(city + region, name, "name")
// }
// if(!locationMap){
// locationMap = matchLocation(city, address, "address")
// }
if(locationMap){
DBObject data = new BasicDBObject()
data.put("school_lat1", locationMap.glat)
data.put("school_lng1", locationMap.glng)
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def match2(id, city, region, address, name){
def locationMap = null;
locationMap = matchLocation2(city + region, name, "name")
if(!locationMap){
locationMap = matchLocation2(city, address, "address")
}
if(locationMap){
DBObject data = new BasicDBObject()
data.put("gaode_name", locationMap.name)
data.put("gaode_address", locationMap.address)
if(locationMap.location){
def arrs = locationMap.location.split(",")
data.put("school_lat", arrs[1].toDouble())
data.put("school_lng", arrs[0].toDouble())
}
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def matchLocation(city, address, cat){
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
def map = [:]
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def datas = geocodes.get(0)
if(datas.formatted_address && cat=="name"){
if(datas.formatted_address.contains(address)){
def glocation = datas.location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}else{
return null
}
}else{
def glocation = geocodes.get(0).location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}
}
return map
}
def matchLocation2(city, address, cat){
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
def map = [:]
if (gloJson && gloJson.get("pois")) {
def geocodes = gloJson.get("pois")
def datas = geocodes.get(0)
return datas
}
return null
}
def findAllDatas(){
def q = new BasicDBObject()
// q.put("school_name","苏州市娄葑实验小学")
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class DistrictSchoolDataExport {
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def regionCodeMap = [:]
static main(args) {
def task = new DistrictSchoolDataExport()
// task.run()
task.megre()
}
def megre(){
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
def ids = []
for(def d : datas1){
ids << d['U']
}
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
for(def d : datas2){
def id = d['U']
if(!ids.contains(id)){
datas1 << d
}
}
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
}
def run(){
def city = "苏州"
getRegionCodeByCity(city)
getRegionCodeByCity("合肥")
getRegionCodeByCity("西安")
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts = []
districts.addAll(districts0)
districts.addAll(districts1)
districts.addAll(districts2)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = []
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
def i = 0
for(def k=0;k<districts.size();k++){
final def dist = districts.get(k)
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
if(primarySchools || midSchools){
def info1 = null;
def info2 = null;
def distance1 = 0
def distance2 = 0
if(primarySchools && dist.lng){
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
if(info1 && info1.school_lat2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
if(distance1>3000){
info1 = null
}
}else{
info1 = null
}
}
if(midSchools) {
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
if(info2 && info2.school_lng2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
if(distance2>3000){
info2 = null
}
}else{
info2 = null
}
}
if(info1|| info2){
def row = []
row << cityCodeMap.get(dist.city)
row << regionCodeMap.get(dist.city +"-"+dist.region)
row << dist.name
row << dist.lng/10000000000.0
row << dist.lat/10000000000.0
row << dist.address
if(info1){
row << info1.school_name
row << info1.school_address!=null?info1.school_address:''
row << info1.attribute!=null?info1.attribute:''
row << info1.school_lng2!=null?info1.school_lng2:''
row << info1.school_lat2!=null?info1.school_lat2:''
// row << distance1
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
if(info2){
row << info2.school_name
row << info2.school_address!=null?info2.school_address:''
row << info2.attribute!=null?info2.attribute:''
row << info2.school_lng2!=null?info2.school_lng2:''
row << info2.school_lat2!=null?info2.school_lat2:''
// row << distance2
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
row << ""
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
row << "fjw"
row << dist._id.toString()
row << "对口"
rets << row
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
}
def selectSchool(lng, lat, def schools){
def minSchool = null
def level0 = []//国家
def level1 = []//省重点
def level2 = []//市重点
def level3 = []//区重点
def level4 = []//普通
def level5 = []//其它
for(def s : schools){
if(s.attribute){
if(s.attribute.contains("国家重点")){
level0 << s
}else if(s.attribute.contains("省重点")){
level1 << s
}else if(s.attribute.contains("市重点")){
level2 << s
}else if(s.attribute.contains("区重点")){
level3 << s
}else if(s.attribute.contains("普通")){
level4 << s
}else{
level5 << s
}
}else{
level5 << s
}
}
if(level0){
if(level0.size()==1){
minSchool = level0.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level0)
}
}else if(level1){
if(level1.size()==1){
minSchool = level1.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level1)
}
}else if(level2){
if(level2.size()==1){
minSchool = level2.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level2)
}
}else if(level3){
if(level3.size()==1){
minSchool = level3.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level3)
}
}else if(level4){
if(level4.size()==1){
minSchool = level4.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level4)
}
}else if(level5){
if(level5.size()==1){
minSchool = level5.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level5)
}
}
def rets = []
if(minSchool){
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
}
if(rets.size()>0){
for(def d : rets){
if(!minSchool.school_address){
minSchool.school_address = d.address
}
if(!minSchool.attribute){
minSchool.attribute = d.attribute
}
}
}else{
if(minSchool){
if(minSchool.school_map_lng){
minSchool.school_lng2 = minSchool.school_map_lng
minSchool.school_lat2 = minSchool.school_map_lat
}else if(minSchool.school_lng1){
minSchool.school_lng2 = minSchool.school_lng1
minSchool.school_lat2 = minSchool.school_lat1
}else if(minSchool.school_lng){
minSchool.school_lng2 = minSchool.school_lng
minSchool.school_lat2 = minSchool.school_lat
}
}
}
return minSchool;
}
def findSchoolInfo(def city, def region, def schoolName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", schoolName)
def dbcursor = school_col.find(q)
def rets = []
for(def obj : dbcursor){
rets << obj
}
return rets
}
def selectMinDistanceSchool(def lat, def lng, def schools){
def minSchool = null;
def minDistance = Integer.MAX_VALUE
for(def s1 : schools){
def lng2 = null
def lat2 = null
if(s1.school_map_lng){
lng2 = s1.school_map_lng
lat2 = s1.school_map_lat
}else if(s1.school_lng1){
lng2 = s1.school_lng1
lat2 = s1.school_lat1
}else if(s1.school_lng){
lng2 = s1.school_lng
lat2 = s1.school_lat
}
if(lat2){
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
println s1.district_name +" "+ s1.school_name+" "+distance
if(distance<minDistance){
minDistance = distance
minSchool = s1
minSchool.school_lng2 = lng2
minSchool.school_lat2 = lat2
}
}
}
return minSchool
}
def findPrimarySchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def findMiddleSchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def getRegionCodeByCity(def city){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
}
}
@@ -0,0 +1,182 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.core.util.BaiDuApiUtils
import com.mongodb.BasicDBObject
import com.fangjia.script.util.DbUtils
class FZloationUpdate {
static main(args) {
def task = new FZloationUpdate();
task.run()
}
def run(){
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
def index = 0
def locationMap = getMoreSameLocationDist(districts)
def total = 0
locationMap.each {k,v->
total = updateLocation(v, total);
println "---------------------------------------------------------------------------------------------------------------"
}
}
def updateLocation(def lists, def total){
for(def dist : lists){
def addrDistMap = getLocation(dist, dist.address)
def nameDistMap = getLocation(dist, dist.region + dist.name)
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
if(addrDistMap){
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
def baidu = null
def lat = null
def lng = null
def upldateFlag = ""
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
lat = addrDistMap.lat
lng = addrDistMap.lng
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Address"
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
lng = nameDistMap.lng
lat = nameDistMap.lat
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Name"
}
if(baidu){
total++
def udata = new BasicDBObject()
udata.put("m_date", new Date())
Long baiduLat = baidu.y*10000000000L
Long baiduLng = baidu.x*10000000000L
udata.put("lat2", baiduLat)
udata.put("lng2", baiduLng)
Long lat0 = lat*10000000000L
Long lng0 = lng*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
}
}
}
}
return total
}
def getMoreSameLocationDist(def districts){
def locationMap = [:]
for(def i=0;i<districts.size();i++){
def dist = districts[i]
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def locationKey = lng + "-" + lat
def list = locationMap.get(locationKey)
if(!list){
list = []
locationMap.put(locationKey, list);
}
list << dist
}
def resMap = [:]
locationMap.each{k,v->
if(v.size()>=2){
v.sort{ a,b->
return a.name.size().compareTo(b.name.size())
}
resMap.put(k,v)
}
}
def lastRetMaps = [:]
resMap.each{k,v->
def swname = null
def count = 0
for(def i=0;i<v.size();i++){
def d = v[i]
if(i==0){
if(d.name.contains("期")||d.name.contains("区")){
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
}else{
swname = d.name
}
}else{
if(d.name.contains(swname)||swname.contains(d.name)){
count++
}
}
}
if(count+1!=v.size()){
lastRetMaps.put(k,v)
}
}
return lastRetMaps;
}
def deleteRegion(def s, def region){
def key = ""
if(s.indexOf(region)>-1){
if(s.indexOf(region+"区")>-1){
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
}else if(s.indexOf(region+"县")>-1){
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
}else if(s.indexOf(region+"市")>-1){
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
}
}else{
key = s;
}
return key
}
def getLocation(dist, addr){
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(dist.city, addr);
if(reps && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
if(location){
def locMap = location.split(",")
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
rMap.distance = distance
rMap.level = level
rMap.lat = locMap[1].toDouble()
rMap.lng = locMap[0].toDouble()
rMap.keyword = map.formatted_address
rMap.location = location
rMap.region = map.district
return rMap;
}
}
}
return null;
}
}
@@ -0,0 +1,4 @@
package com.fangjia.script.gj.test
class FzLocationBatch {
}
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
// task.matchHouse();
// task.villa()
task.addressParse()
// task.addressParse()
task.deleteProcess()
}
def repeat(){
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
return floor;
}
def deleteProcess(){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
def addrList = []
for(def d : datas){
def addr = d['G']
addrList << addr
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
def addr = row['C']
if(addrList.contains(addr)){
row['R'] = "已调查"
}
},0);
}
}
@@ -0,0 +1,3 @@
package com.fangjia.script.gj.test
println "wwww"
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202101);
def count2 = findTotalCount(source, 202102);
def count1 = findTotalCount(source, 202102);
def count2 = findTotalCount(source, 202103);
println source +" "+count1 +" "+count2
}
}
@@ -0,0 +1,213 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.groovy.JsonSlurper
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class AddFangSchoolRegion {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddFangSchoolRegion();
//task.process()
//task.match()
task.addMapLocation()
}
def addMapLocation(){
def q = new BasicDBObject()
def dbcursor = district_school_col.find(q)
def index = 0
for(def d : dbcursor){
def data = findSchool(d.city, d.region, d.school_name)
if(data && data.location){
def location = data.location
if(location){
def udata = new BasicDBObject()
udata.put("school_map_lng", location[0])
udata.put("school_map_lat", location[1])
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
index ++
println index
}
}
def match(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
// if(!data.match_name){
println i +" "+data.district_name
matching(data);
// }
} catch (Exception e) {
println "Òì³£"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def matching(def it){
def rMap = matchDistrict(it.city, it.region, it.district_name)
if(!rMap){
rMap = matchDistrict(it.city, it.region, it.address)
}
if(rMap){
DBObject data = new BasicDBObject()
data.put("match_city", rMap.city)
data.put("match_region", rMap.region)
data.put("match_name", rMap.name)
data.put("match_address", rMap.address)
data.put("match_lat", rMap.lat)
data.put("match_lng", rMap.lng)
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
}
}
def matchDistrict(def city, def region, def address){
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dto = doc.result
if(dto && dto.searchAddress){
def ret = dto.matchResult
if(ret.match){
rMap.city = dto.city
rMap.region = dto.district
rMap.name = dto.name
rMap.lat = dto.lat
rMap.lng = dto.lng
rMap.id = dto.id
rMap.address = dto.address
}
}
}
}
return rMap;
}
def process(){
def datas = findNoRegionData();
findSimialData(datas);
}
def findAllDatas(){
def q = new BasicDBObject()
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
def findNoRegionData(){
def q = new BasicDBObject()
q.put("region", null)
def dbcursor = district_school_col.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findSimialData(datas){
for(def d : datas){
def obj = findThirdPriceData(d.city, d.url+"/")
if(!obj){
obj = findThirdPriceByName(d.city, d.district_name)
}
if(obj){
d.region = obj.region
if(!d.district_addrss){
d.district_address = obj.address
}
updateDistrictSchool(d);
}else{
println d.school_name+"" + d.district_name+"û²éµ½"
}
}
}
def findThirdPriceData(def city, def url){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("district_id", url)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def findThirdPriceByName(def city, def name){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("name", name)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def updateDistrictSchool(d){
DBObject data = new BasicDBObject()
data.put("region", d.region)
data.put("district_address", d.district_address)
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
}
def findSchool(city, region, school_name){
DBObject data = new BasicDBObject()
data.put("city", city)
data.put("region", region)
data.put("name", school_name)
return school_col.findOne(data)
}
}
@@ -0,0 +1,344 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.alibaba.fastjson.annotation.JSONField
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.httpclient.NameValuePair
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpEntity
import org.apache.http.client.entity.UrlEncodedFormEntity
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpPost
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import java.nio.charset.Charset
import java.util.zip.GZIPInputStream
class GushuwangCrawl {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
static main(args) {
def task = new GushuwangCrawl();
task.process()
}
void process(Object args) {
// chaxuequCrawl()
suzhouCrawl();
}
def suzhouCrawl(){
def page = 1
def stop = false
def list = []
while(!stop){
def s = schoolList(page);
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.ok){
def datas = jsonObject.extra.list
if(datas){
list.addAll(datas)
page++
println page
}else{
stop = true
}
}else{
stop = true
}
}
}
if(list){
def schools = []
for(def d : list){
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = "苏州"
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(d.schoolname)
schoolData.name = d.schoolname
schoolData.address = d.adr
/**来源网站*/
schoolData.source = "苏州学区查询"
/**房源详情url*/
schools << schoolData
}
saveChaxuequSchool(schools)
}
}
def chaxuequCrawl(){
def page = 1
def stop = false
def schools = []
while(!stop){
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
println "内容:"+s
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject.code == 200){
def data = jsonObject.data
def totalPage = data.pages
if(data.list){
page++
if(page+1>totalPage){
stop = true;
}
schools.addAll(data.list)
}
}
}
}
print(schools.size());
def districtSchools = []
def schoolInfos = []
if(schools.size()>0){
def url = "https://51yongka.com/fapi/school/details"
for(def d : schools){
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
def s = chaxuequInfoCrawl(url, params)
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.code==200){
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
def districts = jsonObject.data.nearbyPropertyList
for(def dist : districts){
def dataMap = new BasicDBObject()
dataMap.city = jsonObject.data.city
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
dataMap.category = converCategory(jsonObject.data.schoolName)
dataMap.district_name = dist.name.trim()
dataMap.district_address = dist.address
dataMap.school_name = jsonObject.data.schoolName.trim()
dataMap.school_address = jsonObject.data.address
dataMap.source = "查学区房"
dataMap.url = url + "?"+params
dataMap.telephone = jsonObject.data.officialTelephone;
districtSchools << dataMap
}
}else{
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = jsonObject.data.city
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(jsonObject.data.schoolName)
schoolData.name = jsonObject.data.schoolName.trim()
schoolData.address = jsonObject.data.address
/**来源网站*/
schoolData.source = "查学区房"
/**房源详情url*/
schoolData.telephone = jsonObject.data.officialTelephone;
schoolInfos << schoolData
println jsonObject.data.schoolName +" 无周边楼盘"
}
}
println s;
}
}
println "小区:"+districtSchools.size()
// saveChaxuequDistrictSchool(districtSchools)
saveChaxuequSchool(schoolInfos)
}
def saveChaxuequSchool(schoolInfos){
for(def ds : schoolInfos){
def obj = findSchoolData(ds.city, ds.school_name);
if(!obj){
school_col.save(ds)
}
}
}
def saveChaxuequDistrictSchool(def districtSchools){
for(def ds : districtSchools){
def obj = findDistrictSchoolData(ds.city, ds.school_name);
if(!obj){
district_school_col.save(ds)
}
}
}
def findSchoolData(city, school_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
return school_col.findOne(q)
}
def findDistrictSchoolData(city, school_name, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
q.put("district_name", district_name)
return district_school_col.findOne(q)
}
def converCategory(def schoolName){
if(schoolName){
if(schoolName.contains("初中部")||schoolName.contains("中学")){
return "初中"
}else if(schoolName.contains("小学")){
return "小学"
}
}
return null
}
def static chaxuequInfoCrawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","51yongka.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","75")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
headers.put("Accept-Encoding","gzip, deflate, br")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
def static schoolList(page) {
def responseMap = [:]
String s = null
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","m.gusuwang.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","40")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
// headers.put("Accept-Encoding","gzip, deflate, br")
//params信息
Map<String, String> params = new HashMap<String, String>();
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
// //post请求
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
} catch (Exception e) {
e.printStackTrace()
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
outputStream.write(content.getBytes());
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
// byte[] data = new byte[1024];
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
// while (inputStream.read(data) != -1) {
// String s = new String(data, Charset.forName("utf-8"));
// sb.append(s);
// }
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
}
@@ -52,7 +52,7 @@ class ElasticUtils {
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
@@ -7,14 +7,14 @@ import java.util.regex.Pattern;
*/
public interface AddressParserPatterns {
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
Pattern STREET = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?");
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零|一三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
Pattern STAGE_PATTERN = Pattern.compile("(([零一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([零一二三四五六七八九十百千\\d]+)期");
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|零|一|二|三|四|五|六|七|八|九|十)+(区|坊)");
Pattern TOWN = Pattern.compile("^.+?(乡|镇|街道)+");
Pattern NUMBER = Pattern.compile("\\d+");
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
Pattern DISTRICT = Pattern.compile("^.{2,}?(区|县)");
}
@@ -1,6 +1,7 @@
package com.fangjia.service.address;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.CNNMFilter;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.core.util.StringUtil;
@@ -17,6 +18,7 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors;
import com.fangjia.service.address.processor.AddressProcessors;
import com.fangjia.service.api.AliyunService;
import com.fangjia.service.bo.AliyunBo;
import com.fangjia.service.bo.MatchAddressScore;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.mongo.addrepo.AddressRepoService;
import com.fangjia.service.mongo.bkoff.HouseLoudongService;
@@ -170,7 +172,9 @@ public class AddressParserServiceImpl implements AddressParserService {
if(aliyunBo != null){
String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", "");
if(StringUtils.isNotBlank(poi)){
addr.setKeywords(poi);
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
addr.setKeywords(poi);
}
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setSuggestCity(aliyunBo.getCity());
@@ -189,6 +193,45 @@ public class AddressParserServiceImpl implements AddressParserService {
if(params.isUseAliyunApi() && aliyunBo != null){
completBuildingField(matchedBuilding, aliyunBo, params);
}
Address tempAddr = matchedBuilding.getSearchAddress();
if(StringUtils.isBlank(tempAddr.getProvince()) || StringUtils.isBlank(tempAddr.getCity()) || StringUtils.isBlank(tempAddr.getDistrict())|| StringUtils.isBlank(tempAddr.getStreet())){
// List<AddressMatchMetaData> metaDatas = addressRepoService.findAllByKeyword(null, params.getBuildingType(), new ArrayList<String>() {{add(tempAddr.getOrgAddress());}}, tempAddr.getLng(), tempAddr.getLat(), 100, false);
tempAddr.addKeyword(tempAddr.getOrgAddress());
List<AddressMatchMetaData> metaDatas = findMatchMetaData(tempAddr, params.getBuildingType(), MetaDataType.MAP, false);
// String addressStr = (StringUtils.isBlank(tempAddr.getStreet())?"":tempAddr.getStreet()) + (StringUtils.isBlank(tempAddr.getStreetNumber())?"":tempAddr.getStreetNumber());
Map<String,List<AddressMatchMetaData>> metaDataMap = new HashMap<String, List<AddressMatchMetaData>>();
if(StringUtils.isBlank(tempAddr.getDistrict()) && !metaDatas.isEmpty()){
for(AddressMatchMetaData metaData:metaDatas){
if(StringUtils.isNotBlank(metaData.getMapCity())){
List<AddressMatchMetaData> metaList = metaDataMap.containsKey(metaData.getMapCity()) ? metaDataMap.get(metaData.getMapCity()) : new ArrayList<AddressMatchMetaData>();
metaList.add(metaData);
metaDataMap.put(metaData.getMapCity(),metaList);
}
}
String tempCity = "";
if(StringUtils.isNotBlank(tempAddr.getCity()) && metaDataMap.containsKey(tempAddr.getCity())){
tempCity = tempAddr.getCity();
}else{
int size = 0;
for(String key:metaDataMap.keySet()){
if(metaDataMap.get(key).size() > size){
size = metaDataMap.get(key).size();
tempCity = key;
}
}
}
if(StringUtils.isNotBlank(tempCity)){
AddressMatchMetaData meta = metaDataMap.get(tempCity).get(0);
tempAddr.setProvince(meta.getMapProvince());
tempAddr.setCity(meta.getMapCity());
tempAddr.setDistrict(meta.getDistrict());
if((tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){
matchedBuilding.setAddress(meta.getAddress());
}
}
}
}
matchedBuilding.setSearchAddress(tempAddr);
return matchedBuilding;
}
@@ -224,12 +267,25 @@ public class AddressParserServiceImpl implements AddressParserService {
if(StringUtils.isBlank(addr.getTownship())){
addr.setTownship(aliyunBo.getTown());
}
addr.setOrgBuildingNumber(aliyunBo.getHouseno());
/*addr.setOrgBuildingNumber(aliyunBo.getHouseno());
addr.setBuildingNumber(aliyunBo.getHouseno());
addr.setOrgUnitNumber(aliyunBo.getCellno());
addr.setUnitNumber(aliyunBo.getCellno());
addr.setOrgRoomNumber(aliyunBo.getRoomno());
addr.setRoomNumber(aliyunBo.getRoomno());
addr.setUnitNumber(aliyunBo.getCellno());*/
if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(²ã|Â¥)$")){
addr.setRoomNumber(aliyunBo.getRoomno());
addr.setRoomNumber(aliyunBo.getRoomno());
}else{
addr.setRoomNumber(null);
addr.setRoomNumber(null);
}
/*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", "");
String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", "");
if(orgRoom!=null && orgRoom.equals(aliRoom)){
addr.setRoomNumber(aliyunBo.getRoomno());
}else{
addr.setRoomNumber(null);
}*/
if(StringUtils.isBlank(addr.getStreet())){
addr.setStreet(aliyunBo.getRoad());
@@ -362,7 +418,7 @@ public class AddressParserServiceImpl implements AddressParserService {
* @param metaDataType
* @return
*/
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType){
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType, boolean validCity){
if(MetaDataType.MAP == metaDataType){
List<String> keywords = address.getKeywordList();
if(keywords.isEmpty()){
@@ -375,7 +431,7 @@ public class AddressParserServiceImpl implements AddressParserService {
}
keywords.add(standardAddress);
}
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100);
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100, validCity);
}else{
return SEAWEED_LOCAL_CACHE.get(new MetaDataCacheKey(address.getCity(), buildingType, MetaDataType.CPDB));
}
@@ -517,20 +573,20 @@ public class AddressParserServiceImpl implements AddressParserService {
scoreResult.setOrgAddress(addr.getOrgAddress());
if(buildingType != null) {
if(MetaDataType.MAP == metaDataType){
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP);
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB));
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP, true);
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true));
}else{
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB);
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true);
}
} else {
if(MetaDataType.MAP == metaDataType){
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP);
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP, true);
}
for (BuildingType bt : addr.guessBuildType()) {
if(metaDatas == null){
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB);
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB, true);
}else{
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB));
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB, true));
}
}
}
@@ -967,7 +1023,12 @@ public class AddressParserServiceImpl implements AddressParserService {
formattedAddressBuilder.append(address.getStreetNumber());
}
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
formattedAddressBuilder.append(building.getAddress());
Address orgAddr = new Address();
orgAddr.setOrgAddress(building.getAddress());
Address addr = AddressProcessors.process(orgAddr);
building.getSearchAddress().setStreet(addr.getStreet());
building.getSearchAddress().setStreetNumber(addr.getStreetNumber());
formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber()));
}
if(StringUtils.isNotBlank(building.getSuggestName())){
formattedAddressBuilder.append(building.getSuggestName());
@@ -985,16 +1046,16 @@ public class AddressParserServiceImpl implements AddressParserService {
&& address.getStreetNumber().endsWith(address.getOrgBuildingNumber())){
}else{
formattedAddressBuilder.append(address.getBuildingNumber());
// formattedAddressBuilder.append(address.getBuildingNumber() + "¶°");
// formattedAddressBuilder.append(address.getBuildingNumber());
formattedAddressBuilder.append(address.getBuildingNumber() + "¶°");
}
}
if(StringUtils.isNotBlank(address.getOrgUnitNumber())){
formattedAddressBuilder.append(address.getUnitNumber());
// formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª");
// formattedAddressBuilder.append(address.getUnitNumber());
formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª");
}
if(StringUtils.isNotBlank(address.getOrgRoomNumber())){
if(StringUtils.isNotBlank(address.getRoomNumber())){
formattedAddressBuilder.append(address.getRoomNumber());
// formattedAddressBuilder.append(address.getRoomNumber() + "ÊÒ");
}
@@ -259,7 +259,38 @@ public class AddressFormatterFactory {
@Override
protected String doFormat(String address) {
String matchStr = DataUtil.match(address, TARGET);
String formatStr = matchStr + "ºÅ";
String formatStr = matchStr.replaceAll("段-","段");
if(!matchStr.endsWith("号")){
formatStr += "号";
}
return address.replace(matchStr,formatStr);
}
});
/**
* 剑南路东段-12号-18号
*/
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))")) {
final Pattern TARGET = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))");
@Override
protected String doFormat(String address) {
String matchStr = DataUtil.match(address, TARGET);
String formatStr = matchStr;
if(!matchStr.endsWith("号")){
formatStr += "号";
}
return address.replace(matchStr,formatStr);
}
});
/**
* 剑南路东段-12号-18号
*/
formatters.add(new AbstractAddressFormatter(Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))")) {
final Pattern TARGET = Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))");
@Override
protected String doFormat(String address) {
String matchStr = DataUtil.match(address, TARGET);
String formatStr = matchStr.replace("-","");
return address.replace(matchStr,formatStr);
}
});
@@ -44,6 +44,24 @@ public class AddressMatchMetaData {
private Double lat;
private String map;
public String getMapProvince() {
return mapProvince;
}
public void setMapProvince(String mapProvince) {
this.mapProvince = mapProvince;
}
public String getMapCity() {
return mapCity;
}
public void setMapCity(String mapCity) {
this.mapCity = mapCity;
}
private String mapProvince;
private String mapCity;
public void addName(String name, float priority){
if(StringUtils.isNotBlank(name)){
String formatName = AddressParserUtil.standardAddressAndName(name);
@@ -50,18 +50,23 @@ public class CityProcessor extends AbstractAddressProcessor {
return RepeatStatus.NEXT;
}
}
if(StringUtils.isNotBlank(city)){
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
String formatCity = RegionData.formatCity(city);
String possibleCity = null;
String possibleCity = parseCityFromAddress(orgAddress, address);
String confirmCity = null;
/**如果省份和城市信息相互矛盾 则用区域推断城市*/
if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){
List<String> possibleCities = speculateCityNameByRegion(address, city);
if(isUnique(possibleCities)){
formatCity = getFirst(possibleCities);
confirmCity = formatCity;
}
}
if(null != formatCity){
city = formatCity;
if(formatCity == null || !formatCity.equals(possibleCity)){
confirmCity = possibleCity;
}
if(null != confirmCity){
city = confirmCity;
address.setCity(city);
address.setCityStatus(CityStatus.SURE);
}
@@ -98,7 +103,7 @@ public class CityProcessor extends AbstractAddressProcessor {
address.setCity(city);
}
if(StringUtils.isNotBlank(city) && scoreRegion(city, orgAddress) > 0.4){
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?" , "");
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?(?!(路|街|大道|巷|道|段))" , "");
address.setAddress(orgAddress);
address.setCityStatus(CityStatus.SURE);
return RepeatStatus.NEXT;
@@ -166,7 +171,7 @@ public class CityProcessor extends AbstractAddressProcessor {
String street = getStreet(address);
for(String city : allCity){
if(street != null){
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)"), street);
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
if(matchResult !=null){
address = address.replace(matchResult.matchContent, "");
}
@@ -199,6 +204,10 @@ public class CityProcessor extends AbstractAddressProcessor {
}
Map<String, List<String>> regionMap = RegionData.getAllRegions(city);
for(String region : regionMap.keySet()){
MatchResult matchResult = matchFirst(Pattern.compile(region + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
if(matchResult !=null){
address = address.replace(matchResult.matchContent, "");
}
if(address.indexOf(region, index) > -1){
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
weight.score += 2;
@@ -24,9 +24,9 @@ public class ProvinceProcessor extends AbstractAddressProcessor {
String orgAddress = address.getAddress();
String[] provinceInfo = getProvince(orgAddress);
if(provinceInfo == null && address.getCity() != null){
/*if(provinceInfo == null && address.getCity() != null){
provinceInfo = getProvince(address.getCity());
}
}*/
if (null != provinceInfo) {
String province = provinceInfo[NAME_IDX];
if (provinceInfo[FULL_NAME_IDX] != null) {//全称匹配
@@ -18,6 +18,7 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
static final Pattern NUMBER_END = Pattern.compile("(?<=\\D)\\d{3,4}$");
static final Pattern HOUSE_NUMBER = Pattern.compile("\\d{3,4}");
static final String HOUSE_UNITS_PATTERN = "(ÊÒ|·¿|ºÅ·¿|ºÅ(?!Ôº)|»§|ÆÌ)+";
static final Pattern HOUSE_PATTERN_CASE02 = Pattern.compile("(\\d+(ÊÒ|·¿|ºÅ)?[¡¢]?)+$");
@Override
public RepeatStatus process(Address address) {
@@ -27,10 +28,17 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
address.setOrgRoomNumber(matchResult.matchContent);
address.setAddress(orgAddress);
analyFloorUseRoomNumber(address);
// analyFloorUseRoomNumber(address);
return RepeatStatus.NEXT;
}
MatchResult matchResult2 = matchFirst(HOUSE_PATTERN_CASE02, orgAddress);
if (matchResult2 != null) {
orgAddress = removeSubstring(orgAddress, matchResult2.start, matchResult2.end);
address.setOrgRoomNumber(matchResult2.matchContent);
address.setAddress(orgAddress);
// analyFloorUseRoomNumber(address);
return RepeatStatus.NEXT;
}
matchResult = matchFirst(HOUSE_PATTERN, orgAddress);
if (matchResult != null) {
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
@@ -39,10 +47,10 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
} else {
address.setOrgRoomNumber(matchResult.matchContent + "ÊÒ");
}
analyFloorUseRoomNumber(address);
// analyFloorUseRoomNumber(address);
address.setAddress(orgAddress);
}
analyFloorUseRoomNumber(address);
// analyFloorUseRoomNumber(address);
removeHouseUnit(address);
return RepeatStatus.NEXT;
}
@@ -41,6 +41,7 @@ public class TooltipProcessor extends AbstractAddressProcessor {
offset += matchResult.matchContent.length();
if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){
address.setTooltip(tooltip);
address.setStreet(getStreet(tooltip));
} else if(containsDistrictName(tooltip)){
address.setTooltip(tooltip);
} else if(RegionData.isRegion(address.getCity(), tooltip)){
@@ -1,6 +1,7 @@
package com.fangjia.service.address.processor;
import com.fangjia.service.address.mode.Address;
import com.fangjia.service.cnAnalyzer.Token;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
@@ -14,6 +15,15 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
@Override
public AbstractAddressProcessor.RepeatStatus process(Address address) {
String orgAddress = address.getAddress();
if(address.getTokens()!=null){
for(Token token:address.getTokens()){
if("district".equals(token.getKeywordTypes())){
address.setKeywords(token.getName());
address.setAddress(orgAddress.replace(token.getName(),""));
return RepeatStatus.NEXT;
}
}
}
List<String> names = tokenDistrictNames(orgAddress);
List<String> kws = new ArrayList<String>();
for(String name : names){
@@ -68,7 +68,7 @@ public interface AmapPoiService {
* @param types
* @return
*/
List<AmapPoi> search(String city, String keywords, Integer limit, String types);
List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity);
/**
@@ -1,5 +1,6 @@
package com.fangjia.service.api;
import com.fangjia.core.ProvinceData;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
@@ -258,14 +259,14 @@ public class AmapPoiServiceImpl implements AmapPoiService {
}
@Override
public List<AmapPoi> search(String city, String keywords, Integer limit, String types) {
public List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity) {
Map<String, String> params = new HashMap<String, String>();
String area = city;
params.put("city", area);
params.put("output", "json");
params.put("keywords", keywords);
params.put("key", "4ccfe3c898a9489884bd2bcb1ef6dcde");
params.put("citylimit", "true");
// params.put("citylimit", "true");
params.put("offset", limit == null?"25":String.valueOf(limit));
if(types != null){
params.put("types", "120301&120302&120201&060101");
@@ -351,15 +352,17 @@ public class AmapPoiServiceImpl implements AmapPoiService {
cityname = poi.getString("cityname");
}
cityname = RegionData.formatCity(cityname);
if(!city.equals(cityname)){
if((city!=null && !city.equals(cityname) && validCity) || cityname == null){
continue;
}
pname = ProvinceData.formatProvince(pname);
AmapPoi amapPoi = new AmapPoi();
amapPoi.setCity(city);
amapPoi.setCity(cityname);
amapPoi.setPname(pname);
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
adname = poi.getString("adname");
String region = RegionData.formatRegion(city, adname);
if(RegionData.isRegion(city, region)){
String region = RegionData.formatRegion(cityname, adname);
if(RegionData.isRegion(cityname, region)){
amapPoi.setRegion(region);
}
}
@@ -380,8 +383,8 @@ public class AmapPoiServiceImpl implements AmapPoiService {
amapPoi.setAlias(alias);
}
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
String region = RegionData.formatRegion(city, poi.getString("adname"));
if(RegionData.isRegion(city, region)){
String region = RegionData.formatRegion(cityname, poi.getString("adname"));
if(RegionData.isRegion(cityname, region)){
amapPoi.setRegion(region);
}
}
@@ -391,7 +394,7 @@ public class AmapPoiServiceImpl implements AmapPoiService {
.replace(cityname, "").replace(adname, "");
amapPoi.setAddress(AddressParserUtil.optimizeAddress(address));
}
amapPoi.setCity(city);
amapPoi.setCity(cityname);
if(poi.containsKey("location") && poi.get("location") instanceof String){
String[] location = poi.getString("location").split(",");
amapPoi.setLng(Double.valueOf(location[0]));
@@ -37,7 +37,7 @@ public interface AddressRepoService {
* @param max
* @return
*/
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max);
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity);
/**
*
@@ -87,7 +87,7 @@ public class AddressRepoServiceImpl implements AddressRepoService {
return addrs;
}
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max){
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity){
String types = null;
String tag = null;
String category = null;
@@ -104,7 +104,11 @@ public class AddressRepoServiceImpl implements AddressRepoService {
tag = "°Ù»õÉ̳¡";
category = "×ÛºÏÉ̳¡";
}
List<AmapPoi> pois = amapPoiService.search(city, StringUtils.join(keywords, "|"), 20, types);
List<AmapPoi> pois = new ArrayList<AmapPoi>();
for(String keyword:keywords){
List<AmapPoi> poiList = amapPoiService.search(city, keyword, 20, types, validCity);
pois.addAll(poiList);
}
List<AddressMatchMetaData> matchMetaDatas = new ArrayList<AddressMatchMetaData>(pois.size());
for(AmapPoi poi: pois){
AddressMatchMetaData data = new AddressMatchMetaData();
@@ -121,6 +125,8 @@ public class AddressRepoServiceImpl implements AddressRepoService {
data.setType(poi.getType());
data.setMap("¸ßµÂ");
data.setCity(city);
data.setMapCity(poi.getCity());
data.setMapProvince(poi.getPname());
if(poi.getAlias() != null){
for(String alia : poi.getAlias()){
data.addName(alia, MatchText.ALIAS_PRIORITY);
@@ -31,10 +31,15 @@ public interface SeawaterService {
void deleteById(String id);
/**
* 批量删除,英文逗号分割,如1,2,3
* @param ids
*/
void deleteByIds(String ids);
@Deprecated
List<Seawater> findAllSeawater(String category, String city, String name, String region, Integer room, String root, String sort, Integer startDate, Integer endDate, int start, int limit, boolean order, String[] fields);
@Deprecated
int findAllSeawaterCount(String category, String city, String name, String region, Integer room, String root, Integer startDate, Integer endDate);
}
@@ -12,10 +12,7 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.client.solrj.util.ClientUtils;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
public class SeawaterServiceImpl implements SeawaterService {
private SeawaterDao seawaterDao;
@@ -188,6 +185,12 @@ public class SeawaterServiceImpl implements SeawaterService {
seawaterDao.deleteById(id);
}
@Override
public void deleteByIds(String ids) {
String[] idArr = ids.split(",");
seawaterDao.deleteByIds(Arrays.asList(idArr));
}
public void setSeawaterDao(SeawaterDao seawaterDao) {
this.seawaterDao = seawaterDao;
}
+2 -2
View File
@@ -29,7 +29,7 @@
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-core</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -154,7 +154,7 @@
<dependency>
<groupId>com.poseidon</groupId>
<artifactId>poseidon-message</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.1</version>
<exclusions>
<exclusion>
<groupId>org.mongodb</groupId>