diff --git a/projects/chafangtong-service/pom.xml b/projects/chafangtong-service/pom.xml index c38bee1c1..d2c64b841 100644 --- a/projects/chafangtong-service/pom.xml +++ b/projects/chafangtong-service/pom.xml @@ -29,7 +29,7 @@ org.sothis sothis-core - 0.2.0-SNAPSHOT + 0.2.0 provided @@ -154,7 +154,7 @@ com.poseidon poseidon-message - 0.0.1-SNAPSHOT + 0.0.1 org.mongodb diff --git a/projects/fangjia-bkoff/pom.xml b/projects/fangjia-bkoff/pom.xml index 28b93ccc1..41b8a352d 100644 --- a/projects/fangjia-bkoff/pom.xml +++ b/projects/fangjia-bkoff/pom.xml @@ -99,7 +99,7 @@ org.sothis sothis-web - 0.2.0-SNAPSHOT + 0.2.0 org.springframework diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java index 1ea3ac457..a223c1674 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java @@ -12,6 +12,7 @@ import com.fangjia.service.mongo.fangjia.SeawaterService; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; import org.sothis.core.util.Pager; +import org.sothis.web.mvc.annotation.Param; import java.io.UnsupportedEncodingException; import java.util.Arrays; @@ -116,6 +117,14 @@ public class SeawaterController { return ret; } + public Object batchDeleteAction(@Param(name="ids") String ids) throws UnsupportedEncodingException { + JsonModel ret = new JsonModel(); + if (StringUtils.isNotBlank(ids)){ + seawaterService.deleteByIds(ids); + } + return ret; + } + public void setSeawaterService(SeawaterService seawaterService) { this.seawaterService = seawaterService; } diff --git a/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl b/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl index 97bc38d1f..69d4cd0f7 100644 --- a/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl @@ -118,9 +118,13 @@ - 查询到${pager.totalRows!}�记录 +
+ 查询到${pager.totalRows!}�记录 + +
+ @@ -142,6 +146,7 @@ <#if seawaterList??> <#list seawaterList as it> +
全选 类型 区域 �区�
<#if it.category! == 'sell'> 二手房 @@ -249,7 +254,7 @@ } }); } - + changeCity('${cityDisplay!}'); $(".hrefClick").click(function(){ @@ -336,8 +341,48 @@ $("#roomSpan").hide(); } } - - + + // 全选 + $("#checkAll").click(function(){ + $('input[name="houseCheck"]').each(function() { + var _this = this + if (_this.checked == true) { + _this.checked = false + } else { + _this.checked = true + } + + }) + }) + + $("#batchDelete").click(function(){ + if(confirm('确认è¦�删除所选的数æ�®å�—?')) { + var ids = ""; + $('input:checkbox[name=houseCheck]:checked').each(function() { + var _this = $(this); + ids += _this.val() + "," + }); + if (ids == "") { + alert("请选择数æ�®") + return false; + } + jQuery.ajax({ + type: 'POST', + url: '../seawater/batchDelete', + data:{ids:ids}, + dataType: 'json', + timeout: 100000, + success: function(model) { + alert("批é‡�删除æ“�作æˆ�功"); + location.reload(); + }, + error : function() { + //alert("æ“�作失败"); + } + }); + } + }); + \ No newline at end of file diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/ElasticDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/ElasticDaoImpl.java index 0e45edde5..ea6749194 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/ElasticDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/ElasticDaoImpl.java @@ -205,7 +205,6 @@ public abstract class ElasticDaoImpl extends AbstractJpaC @Override public void deleteByIds(List idList) { - } @Override diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java index dd28f68b7..edbd0546e 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java @@ -14,10 +14,8 @@ public interface SeawaterDao extends com.fangjia.dal.SeawaterDao { Seawater findByIdFromEs(String id,String table); - List findSeawatersFromEs(String category, String city, String region, String name, List alias, List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root, String sort, boolean order, int start, int limit , String table); List findHouses(String city, String region, List alias, Integer minPrice, Integer maxPrice, Integer limit); - } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 5fac9cbef..3a6029b19 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -96,6 +96,44 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i return new ArrayList(); } + @Override + public void deleteById(String id) { + deleteByIds(Arrays.asList(id)); + } + + private Map buildHeader() { + Base64 token = new Base64(); + String authenticationEncoding = token.encodeAsString(new String("" + ES_USER + ":" + ES_PASSWORD + "").getBytes()); + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Basic " + authenticationEncoding + ""); + return headers; + } + + @Override + public void deleteByIds(List idList) { + HttpRequest request = new HttpRequest(); + + JSONObject reqBody = new JSONObject(); + JSONObject query = new JSONObject(); + JSONObject bool = new JSONObject(); + JSONArray must = new JSONArray(); + must.add(addTerms("_id", idList)); + bool.put("must", must); + query.put("bool", bool); + reqBody.put("query", query); + + JSONObject script = new JSONObject(); + script.put("source", "ctx._source.status=1"); + reqBody.put("script", script); + + try { + request.postWithBody("http://es.fangjia.com:9200/seawater/_update_by_query", reqBody, this.buildHeader(), "UTF-8"); + } catch (HttpRequestException e) { + e.printStackTrace(); + } + } + private JSONObject addMatch(String name, String value){ JSONObject term = new JSONObject(); JSONObject match = new JSONObject(); @@ -338,6 +376,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(limit >= 0){ reqBody.put("size", limit); } + + // ²»Õ¹Ê¾É¾³ý + JSONObject param = new JSONObject(); + param.put("status", 0); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + query.put("must", jsonList); q.put("bool", query); reqBody.put("query", q); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index d9287fdaf..de82fe219 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1790,6 +1790,7 @@ public class AddressController { } } dto.append("buildingNumber", address.getBuildingNumber()); + dto.append("floor", address.getFloor()); dto.append("unitNumber", address.getUnitNumber()); dto.append("roomNumber", address.getRoomNumber()); dto.append("formattedAddress", address.getFormattedAddress()); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8914794b4..34a65ee51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -218,7 +218,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "¹¤ÉÌ´óÂ¥"); +// q.put("name","¶«Ô¢Ð´å") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy index cbf7ef9e6..f1f8918e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy @@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{ houseLoudng.fitment_cost = ret.fitment_charge houseLoudng.car_port = ret.carport houseLoudng.house_height = ret.house_hight - houseLoudng.worry_facility" : null, - houseLoudng.fitment_age = ret.fitment_age - houseLoudng.lighting_ventilation" : null, - houseLoudng.build_type = ret.build_type - houseLoudng.house_property" : null, - houseLoudng.noise" : null, - houseLoudng.sight" : null, - houseLoudng.fitment_rank" : null, - houseLoudng.sideset" : null, +// houseLoudng.worry_facility" : null, +// houseLoudng.fitment_age = ret.fitment_age +// houseLoudng.lighting_ventilation" : null, +// houseLoudng.build_type = ret.build_type +// houseLoudng.house_property" : null, +// houseLoudng.noise" : null, +// houseLoudng.sight" : null, +// houseLoudng.fitment_rank" : null, +// houseLoudng.sideset" : null, houseLoudng.district_no = district._id diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy index 401002889..fa59d2b45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy @@ -1,20 +1,37 @@ package com.fangjia.script.gj - +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.Connection import java.sql.ResultSet import java.sql.SQLException +import java.sql.Statement +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.regex.Matcher +import java.util.regex.Pattern class EsHouseStat { - public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) +// DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020) + +// public static JdbcDataSource dataSource = new JdbcDataSource() +// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) static { dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -23,12 +40,88 @@ class EsHouseStat { dataSource.setProperties(connectionProperties) } + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + public static void main(String[] args){ def task = new EsHouseStat(); //task.run(); - task.testRun("³£ÖÝ") +// task.testRun("ÉîÛÚ") // task.exportOpenLog(); +// task.exportMongoLog() // task.exportPoi(); +// task.cityStat() +// task.stat(); +// task.matchStat(); +// task.statNoHouseCalType() +// task.statCityHouse() + +// task.szRentHouseStat(); +// task.addPrice(); + + task.statNoHouseCCBData(); + } + + def addPrice(){ + def datas = [] + def data1 = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐÒøÐÐ\\ÄÂÁ¼Êé\\±±¾©³É¶¼.xlsx",0,1) + def data2 = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐÒøÐÐ\\ÄÂÁ¼Êé\\´óÁ¬º¼ÖÝÏÃÃÅ.xlsx",0,1) + def data3 = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐÒøÐÐ\\ÄÂÁ¼Êé\\ÉϺ£.xlsx",0,1) + def data4 = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐÒøÐÐ\\ÄÂÁ¼Êé\\ÉîÛÚ¹ãÖÝ.xlsx",0,1) + def data5 = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐÒøÐÐ\\ÄÂÁ¼Êé\\ÖØÇìËÕÖÝ.xlsx",0,1) + datas.addAll(data1) + datas.addAll(data2) + datas.addAll(data3) + datas.addAll(data4) + datas.addAll(data5) + + println "data1¹²£º"+data1.size() + println "data2¹²£º"+data2.size() + println "data3¹²£º"+data3.size() + println "data4¹²£º"+data4.size() + println "data5¹²£º"+data5.size() + println "¹²£º"+datas.size() + + final CountDownLatch latch = new CountDownLatch(datas.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + +// for(def data : data1){ +// def dbo = CommonDatautils.queryDistrictPrice(data[0], data[1], data[2], DateUtils.parseFromTightYYYYMMDD("2021-02-07")) +// data << dbo.avg_price +// } + + + for(def i=0;i0){ + results << dist + } + } + ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐÒøÐÐ\\ÄÂÁ¼Êé\\½¨ÐÐ10¸ö³ÇÊм۸ñÊý¾Ý2.xlsx",results,"Êý¾Ý") + } def run(){ @@ -61,6 +154,189 @@ class EsHouseStat { } + def szRentHouseStat(){ + def citys = ["ÉϺ£","±±¾©","¹ãÖÝ","ÉîÛÚ","º¼ÖÝ","ÄϾ©","Ìì½ò","³É¶¼","Î÷°²","ÉòÑô","ÖØÇì","ËÕÖÝ","Î人","³¤É³","Çൺ","ÏÃÃÅ","Äþ²¨","´óÁ¬","ÎÞÎý"] +// def citys = ["³¤É³"] + def rets = [] + rets << ["³ÇÊÐ","Ð¡ÇøÊýÁ¿","2020-08-01","2020-09-01","2020-10-01","2020-11-01","2020-12-01","2021-01-01"] + for(def city : citys){ + def month0 = findRentHouse(city, 20200801, 20210201) +// def month1 = findRentHouse(city, 20200801, 20200901) +// def month2 = findRentHouse(city, 20200901, 20201001) +// def month3 = findRentHouse(city, 20201001, 20201101) +// def month4 = findRentHouse(city, 20201101, 20201201) +// def month5 = findRentHouse(city, 20201201, 20210101) +// def month6 = findRentHouse(city, 20210101, 20210201) + + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name","alias"]) + def map = findMatchDistrictCount(city, districts); + +// def matchCount1 = 0; +// def matchCount2 = 0; +// def matchCount3 = 0; +// def matchCount4 = 0; +// def matchCount5 = 0; +// def matchCount6 = 0; +// for(def dist : districts){ +// def nameQuery = getAllName(dist); +// def month11 = findMatchHouse(city, dist.region, nameQuery, 20200801, 20200901) +// if(month11>0){ +// matchCount1 ++ +// } +// def month21 = findMatchHouse(city, dist.region, nameQuery, 20200901, 20201001) +// if(month21>0){ +// matchCount2 ++ +// } +// def month31 = findMatchHouse(city, dist.region, nameQuery, 20201001, 20201101) +// if(month31>0){ +// matchCount3 ++ +// } +// def month41 = findMatchHouse(city, dist.region, nameQuery, 20201101, 20201201) +// if(month41>0){ +// matchCount4 ++ +// } +// def month51= findMatchHouse(city, dist.region, nameQuery, 20201201, 20210101) +// if(month51>0){ +// matchCount5 ++ +// } +// def month61 = findMatchHouse(city, dist.region, nameQuery, 20210101, 20210201) +// if(month61>0){ +// matchCount6++ +// } + +// rets << [city, districts.size(), month1+"(${map.matchCount1})",month2+"(${map.matchCount2})",month3+"(${map.matchCount3})",month4+"(${map.matchCount4})",month5+"(${map.matchCount5})",month6+"(${map.matchCount6})" ] + rets << [city, districts.size(), month0] + println city +" Íê³É" + } + + ExcelUtils.write("d:\\Êý¾Ý\\×â·¿Êý¾Ýͳ¼Æ1.xlsx",rets, "×ⷿͳ¼Æ") + } + + + + def findMatchDistrictCount(def city, def districts){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + def matchCount0 = 0; + def matchCount1 = 0; + def matchCount2 = 0; + def matchCount3 = 0; + def matchCount4 = 0; + def matchCount5 = 0; + def matchCount6 = 0; + for(def i=0;i0){ + matchCount0 ++ + } +// def month11 = findMatchHouse(city, dist.region, nameQuery, 20200801, 20200901) +// if(month11>0){ +// matchCount1 ++ +// } +// def month21 = findMatchHouse(city, dist.region, nameQuery, 20200901, 20201001) +// if(month21>0){ +// matchCount2 ++ +// } +// def month31 = findMatchHouse(city, dist.region, nameQuery, 20201001, 20201101) +// if(month31>0){ +// matchCount3 ++ +// } +// def month41 = findMatchHouse(city, dist.region, nameQuery, 20201101, 20201201) +// if(month41>0){ +// matchCount4 ++ +// } +// def month51= findMatchHouse(city, dist.region, nameQuery, 20201201, 20210101) +// if(month51>0){ +// matchCount5 ++ +// } +// def month61 = findMatchHouse(city, dist.region, nameQuery, 20210101, 20210201) +// if(month61>0){ +// matchCount6++ +// } + + }catch (Exception e){ + + }finally { + latch.countDown() + } + + } + } + ) + } + latch.await() + + def map = [:] + map.matchCount0 = matchCount0 +// map.matchCount1 = matchCount1 +// map.matchCount2 = matchCount2 +// map.matchCount3 = matchCount3 +// map.matchCount4 = matchCount4 +// map.matchCount5 = matchCount5 +// map.matchCount6 = matchCount6 + return map; + } + + def findMatchHouse(def city, def region, def nameQuery, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and cat = 'rent'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + sqlBuffer.append(" and (" + nameQuery + ")"); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def total = 0; + while (rs.next()){ + total = rs.getInt("count") + } + return total; + } + + def getAllName(dist){ + Set aliasNames = new LinkedHashSet(); + aliasNames.add(dist.name); + if (dist != null && dist.alias != null) { + aliasNames.addAll(dist.alias); + } + + List namePars = new ArrayList(); + if(aliasNames!=null && aliasNames.size()>0) { + for(String name : aliasNames){ + if(StringUtils.isNotBlank(name) && name.trim().length() > 1){ + namePars.add("district_name ='" + name + "'"); + } + } + } + + if(namePars.size() > 0){ + return StringUtils.join(namePars, " OR ") + } + return null; + } + + def findRentHouse(def city, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'rent'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def total = 0; + while (rs.next()){ + total = rs.getInt("count") + } + return total; + } + def findHouse(def city, def startDate, def endDate){ final StringBuffer sqlBuffer = new StringBuffer() sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") @@ -116,46 +392,344 @@ class EsHouseStat { def findLogs3(){ - //QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007736')).must(QueryBuilders.termQuery("request_uri", '/property/detail')) - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007713')).must(QueryBuilders.termQuery("request_uri", '/property/evaluate')).must(QueryBuilders.termQuery("code", 200)) - def rets = [] - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); - def curDate = DateUtils.parseFromTightByPattern("2020-07-01", "yyyy-MM-dd") - if(houses){ - for (int k=0;k= curDate.getTime()){// && - def ret = [] - def paramMap = map.parameter_map - def param_id = map.id - ret << param_id - ret << map.company_name - ret << map.ip - ret << map.request_uri - ret << map.c_date.toString().substring(0, map.c_date.indexOf("T")) - ret << map.data_status; - ret << map.code - ret << map.msg + //QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007736')).must(QueryBuilders.termQuery("request_uri", '/property/detail')) + // QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007721')).must(QueryBuilders.termQuery("request_uri", '/property/evaluate')).must(QueryBuilders.termQuery("code", 200)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007751')).must(QueryBuilders.termQuery("request_uri", '/property/evaluateForTest')).must(QueryBuilders.termQuery("parameter_map.city", 'ÉϺ£')) + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def curDate = DateUtils.parseFromTightByPattern("2020-07-01", "yyyy-MM-dd") + if(houses){ + for (int k=0;k= curDate.getTime() + def ret = [] + def paramMap = map.parameter_map + def param_id = map.id + ret << param_id + ret << map.company_name + ret << map.ip + ret << map.request_uri + ret << map.c_date.toString().substring(0, map.c_date.indexOf("T")) + ret << map.data_status; + ret << map.code + ret << map.msg - def paramStr = "" - paramMap.each{key,val-> - if(key!="biz_code"){ - paramStr = paramStr+" "+key+":"+val - } + def paramStr = "" + paramMap.each{key,val-> + if(key!="biz_code"){ + paramStr = paramStr+" "+key+":"+val } - ret << paramStr.toString() - - def resMap = map.res_map - ret << resMap.result.toString() - rets << ret } + ret << paramStr.toString() + def resMap = map.res_map + ret << resMap.result.toString() + rets << ret + } + + } + } + return rets; + } + + + def findLogs4(def city){ + + def time = DateUtils.parseFromTightYYYYMMDD("2021-01-19") + QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007751')).must(QueryBuilders.termQuery("request_uri", '/property/evaluateForTest')).must(QueryBuilders.termQuery("parameter_map.city", city)).must(date_range_cdate) + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + //"³ÇÊÐ","Â¥ÅÌÃû³Æ","µØÖ·", + rets << ["³ÇÊÐ","Ãæ»ý","ÊÒ","Ìü","ÎÀ","Â¥¸ß","ÎïÒµÀàÐÍ","·¿Áä","×°ÐÞ","³¯Ïò","¹À¼Ûʱ¼ä","½á¹ûÐÅÏ¢","³ÇÊÐ","ÇøÓò","Â¥ÅÌÃû³Æ","×ܼÛ","µ¥¼Û","¼ÆË㷽ʽ","·¿Ô´ÁбíÁ¿","½¨ÖþÄê´ú","ÎïÒµÀàÐÍ","¾àÀë","½ü°ëÄêÕǵø·ù","Í¬ÇøÓò20%Ð¡Çø¼Ûλ","Í¬ÇøÓò80%Ð¡Çø¼Ûλ","¸½½üÐ¡ÇøÁ¿","¾­Î³¶È"] + if(houses){ + for (int k=0;k= curDate.getTime() +// if(date.contains("2021-01-19")){ + + def ret = [] + def paramMap = map.parameter_map + //³ÇÊР¥ÅÌÃû³Æ µØÖ· Ãæ»ý ÊÒ Ìü ÎÀ Â¥¸ß ³¯Ïò ×°ÐÞ ·¿Áä ¹À¼Ûʱ¼ä ÎïÒµÀàÐÍ + ret << cDate +// ret << paramMap.name +// ret << paramMap.address + ret << paramMap.size + ret << paramMap.room + ret << paramMap.hall + ret << paramMap.toilet + ret << paramMap.maxFloor + ret << paramMap.propertyType + ret << paramMap.age + ret << paramMap.decoration + ret << paramMap.direction + ret << paramMap.calDate + + def resMap = map.res_map.result +// println resMap +// if(!resMap){ +// println "sss" +// } + if(!resMap){ + ret << map.res_map.msg + }else{ + if(resMap.houses){ + //³ÇÊÐ ÇøÓò Â¥ÅÌÃû³Æ ×Ü¼Û µ¥¼Û ¼ÆË㷽ʽ ·¿Ô´Áбí + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << resMap.houses.size() + }else{ + //³ÇÊÐ ÇøÓò Â¥ÅÌÃû³Æ ×Ü¼Û µ¥¼Û ¼ÆË㷽ʽ Ä£ÐÍÌØÕ÷£¨¾­Î³¶È£¬½¨ÖþÄê´ú£¬¸½½üÐ¡Çø£¬¾àÀ룬½ü°ëÄêÕǵø·ù£¬Í¬ÇøÓò20%£¬80%¼Û룩 + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << 0 + ret << resMap.buildYear + ret << resMap.modelFeatures.propertyType + ret << resMap.modelFeatures.distance + ret << resMap.modelFeatures.halfYearMarkup + ret << resMap.modelFeatures.region20PercentPrice + ret << resMap.modelFeatures.region80PercentPrice + ret << resMap.modelFeatures.nearCommunity.size() + ret << resMap.modelFeatures.lat+" "+resMap.modelFeatures.lng + + rets << ret + + } + } + +// }else{ +// println map.c_date +// } + } + } + return rets; + } + + def findTotalCount(def city) { + + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + def count = LOG.count(q) + return count; + + } + + def findLogs6(def city, def vals) { + + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + q.put("msg", new BasicDBObject('$in', vals)) + def count = LOG.count(q) + return count; + + } + + def findHouseType(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q, keys) + def count = 0 + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + if(resMap.calcType==value){ + count ++; } } - return rets; + } + return count; + } + + + def findHousePriceType(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def count = 0 + BigDecimal totalPrice = 0 + BigDecimal totalArea = 0 + BigDecimal totalAvgPrice = 0 + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + def price = resMap.totalPrice + def area = d.parameter_map.size + if(resMap.calcType==value){ + count ++; + totalPrice = totalPrice + price + totalArea = totalArea + area.toDouble() + } + } + } + + def map = [:] + map.totalPrice = totalPrice + map.avgPrice = (totalPrice/totalArea).toInteger() + return map; + + } + + + def findLogs5(def city){ + + def q = new BasicDBObject() + q.put("user_id","1007751") + q.put("parameter_map.city",city) + q.put("request_uri" ,"/property/evaluateForTest") +// q.put("c_date" ,new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-19"))) + def rets = [] +// rets << ["³ÇÊÐ","Â¥ÅÌÃû³Æ","µØÖ·","Ãæ»ý","ÊÒ","Ìü","ÎÀ","Â¥¸ß","ÎïÒµÀàÐÍ","·¿Áä","×°ÐÞ","³¯Ïò","¹À¼Ûʱ¼ä","½á¹ûÐÅÏ¢","³ÇÊÐ","ÇøÓò","Â¥ÅÌÃû³Æ","×ܼÛ","µ¥¼Û","¼ÆË㷽ʽ","·¿Ô´ÁбíÁ¿","½¨ÖþÄê´ú","ÎïÒµÀàÐÍ","½ü°ëÄêÕǵø·ù","Í¬ÇøÓò20%Ð¡Çø¼Ûλ","Í¬ÇøÓò80%Ð¡Çø¼Ûλ","¸½½üÐ¡ÇøÁ¿","¾­Î³¶È"] + def houses = LOG.find(q) +// def count = LOG.count(q) + println LOG.count(q) + if(houses){ + def i = 0 + for(def map : houses){ + i++ + println i +" "+map +// def map = houses[k]; +// def date = map.c_date +// def cDate = DateUtils.parseFromTightByPattern(date.toString().substring(0, date.toString().indexOf("T")), "yyyy-MM-dd") +// if(map.code == 200){// && && cDate.getTime()>= curDate.getTime() + if(map.msg=="¸ÃÐ¡Çø²»´æÔÚ" || map.msg=="ÔÝʱ²»Ö§³Ö¸ÃÐ¡ÇøµÄ¹À¼Û"){ + continue + } + def ret = [] + def paramMap = map.parameter_map + //³ÇÊР¥ÅÌÃû³Æ µØÖ· Ãæ»ý ÊÒ Ìü ÎÀ Â¥¸ß ³¯Ïò ×°ÐÞ ·¿Áä ¹À¼Ûʱ¼ä ÎïÒµÀàÐÍ + ret << paramMap.city + ret << paramMap.name + ret << paramMap.address + ret << paramMap.size + ret << paramMap.room + ret << paramMap.hall + ret << paramMap.toilet + ret << paramMap.maxFloor + ret << paramMap.propertyType + ret << paramMap.age + ret << paramMap.decoration + ret << paramMap.direction + ret << paramMap.calDate + + def resMap = map.res_map.result + println resMap + + if(!resMap){ + ret << map.res_map.msg + }else{ + if(resMap.houses){ + //³ÇÊÐ ÇøÓò Â¥ÅÌÃû³Æ ×Ü¼Û µ¥¼Û ¼ÆË㷽ʽ ·¿Ô´Áбí + ret << "" + ret << resMap.city + ret << handleIllegalCharacter(resMap.district) + ret << handleIllegalCharacter(resMap.name) + ret << resMap.totalPrice + ret << resMap.unitPrice + ret << resMap.calcType + ret << resMap.houses.size() + + }else{ + //³ÇÊÐ ÇøÓò Â¥ÅÌÃû³Æ ×Ü¼Û µ¥¼Û ¼ÆË㷽ʽ Ä£ÐÍÌØÕ÷£¨¾­Î³¶È£¬½¨ÖþÄê´ú£¬¸½½üÐ¡Çø£¬¾àÀ룬½ü°ëÄêÕǵø·ù£¬Í¬ÇøÓò20%£¬80%¼Û룩 + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << 0 + ret << resMap.modelFeatures.buildYear + ret << resMap.modelFeatures.propertyType + ret << resMap.modelFeatures.halfYearMarkup + ret << resMap.modelFeatures.region20PercentPrice + ret << resMap.modelFeatures.region80PercentPrice + ret << resMap.modelFeatures.nearCommunity.size() + ret << resMap.modelFeatures.lat+" "+resMap.modelFeatures.lng + + } + } + rets << ret + +// } + + } + } + return rets; + } + + + def findLogs6(def city){ + def q = new BasicDBObject() + q.put("user_id","1007751") + q.put("parameter_map.city",city) + q.put("request_uri" ,"/property/evaluateForTest") + q.put("c_date" ,new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-19"))) + def rets = [] + def houses = LOG.find(q) + if(houses){ + for(def map : houses){ + if(map.msg=="ÔÝʱ²»Ö§³Ö¸ÃÐ¡ÇøµÄ¹À¼Û"){ + continue + } + + def resMap = map.res_map.result + println resMap + if(resMap && map.code == 200 && resMap.calcType==2){ + def ret = [] + //³ÇÊÐ ÇøÓò Â¥ÅÌÃû³Æ ×Ü¼Û µ¥¼Û ¼ÆË㷽ʽ Ä£ÐÍÌØÕ÷£¨¾­Î³¶È£¬½¨ÖþÄê´ú£¬¸½½üÐ¡Çø£¬¾àÀ룬½ü°ëÄêÕǵø·ù£¬Í¬ÇøÓò20%£¬80%¼Û룩 + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + + def paramMap = map.parameter_map + ret << paramMap.calDate + + rets << ret + } + } + } + return rets; + } + + + public static String handleIllegalCharacter(String str) { + str = str.replaceAll("[\\x00-\\x08\\x0b-\\x0c\\x0e-\\x1f]", ""); + str = str.replace("?","#") + str = str.trim(); + return str; } def findPoiLogs(def city){ @@ -165,20 +739,20 @@ class EsHouseStat { ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) def rets = [] while (rs.next()) { - def date = rs.getDate("m_date") - def type = rs.getString("typecode") - if(["141202", "141203"].contains(type)){ - def ret = [] - ret << rs.getString("cityname") - ret << rs.getString("adname") - ret << rs.getString("name") - ret << rs.getString("address") - ret << rs.getString("type") - ret << rs.getString("typecode") - ret << rs.getFloat("location.lon") - ret << rs.getFloat("location.lat") - rets << ret - } + def date = rs.getDate("m_date") + def type = rs.getString("typecode") + if(["141202", "141203"].contains(type)){ + def ret = [] + ret << rs.getString("cityname") + ret << rs.getString("adname") + ret << rs.getString("name") + ret << rs.getString("address") + ret << rs.getString("type") + ret << rs.getString("typecode") + ret << rs.getFloat("location.lon") + ret << rs.getFloat("location.lat") + rets << ret + } } return rets; } @@ -200,7 +774,7 @@ class EsHouseStat { if(hits.hits){ def arr = hits.hits if(arr){ - def source = arr[0]._source + def source = arr[0]._source if(source && source.code == 200){ def ret = [] def parameter_map = source.parameter_map @@ -284,8 +858,19 @@ class EsHouseStat { def exportOpenLog(){ - def rets = findLogs3(); - ExcelUtils.write("D:\\Êý¾Ý\\¶«Ý¸ÒøÐÐ×ÜÐÐ.xlsx", rets,"Êý¾Ý") + def city = "ËÕÖÝ" + def rets = findLogs4(city); + ExcelUtils.write("D:\\Êý¾Ý\\ÉîÛÚ½¨ÐÐÕýʽ("+city+").xlsx", rets,"³É¹¦") + } + def exportMongoLog(){ + //"ÉϺ£","±±¾©","³É¶¼","ÖØÇì","ËÕÖÝ","¹ãÖÝ","´óÁ¬","ÏÃÃÅ","º¼ÖÝ", + def citys = ["ÉîÛÚ"] + for(def city :citys){ + def rets = findLogs5(city); +// def rets = findLogs6(city); + ExcelUtils.write("D:\\Êý¾Ý\\ÉîÛÚ½¨ÐÐ\\ÉîÛÚ½¨ÐÐÕýʽ("+city+").xlsx", rets,"³É¹¦") + } + } def exportPoi(){ @@ -309,4 +894,276 @@ class EsHouseStat { } } + + def cityStat(){ + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("request_uri", "/property/evaluateForTest") + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def map = [:] + for(def d : cursor){ + def resMap = d.parameter_map + if(resMap){ + def count = map.get(resMap.city) + if(!count){ + count = 0 + } + map.put(resMap.city, count+1) + + }else{ + println d + } + } + + println map.toString() + } + + + def stat(){ +// def citys = ["ÉîÛÚ"] + def citys = ["ÉϺ£","±±¾©","³É¶¼","ÖØÇì","ËÕÖÝ","¹ãÖÝ","´óÁ¬","ÏÃÃÅ","º¼ÖÝ","ÉîÛÚ"] + def rets = [] + rets << ["³ÇÊÐ","¹À¼Û×ܵ÷ÓÃÁ¿","³É¹¦µ÷ÓÃÁ¿","ʧ°Üµ÷ÓÃÁ¿","ÓÐÑù±¾µ÷ÓÃÁ¿","ÎÞÑù±¾µ÷ÓÃÁ¿"] + for(def city :citys){ + //def datas = ExcelUtils.read("D:\\Êý¾Ý\\ÉîÛÚ½¨ÐÐ\\ÉîÛÚ½¨ÐÐÕýʽ("+city+").xlsx",0,1) + + def map = findHousePriceType(city, 1); + def ret = [] + ret << city + ret << findTotalCount(city); + ret << findLogs6(city, ["ok"]); + ret << findLogs6(city, ["¸ÃÐ¡Çø²»´æÔÚ","ÔÝʱ²»Ö§³Ö¸ÃÐ¡ÇøµÄ¹À¼Û"]); + ret << findHouseType(city, 1); + ret << findHouseType(city, 2); + + + ret << map.totalPrice + ret << map.avgPrice + println ret + rets << ret + } + + ExcelUtils.write("D:\\Êý¾Ý\\ÉîÛÚ½¨ÐÐ\\ÉîÛÚ½¨Ðе÷ÓÃͳ¼Æ2.xlsx", rets, "ͳ¼Æ") + } + def findSuccess(def datas){ + def count = 0 + for(def data : datas){ + def msg = data['N'] + if(!msg){ + count++ + } + } + return count + } + + def findFail(def datas){ + def count = 0 + for(def data : datas){ + def msg = data['N'] + if(msg=="¸ÃÐ¡Çø²»´æÔÚ" || msg =="ÔÝʱ²»Ö§³Ö¸ÃÐ¡ÇøµÄ¹À¼Û"){ + count++ + } + } + return count + } + + def findHouse(def datas, def value){ + def count = 0 + for(def data : datas){ + def type = data['T'] + if(type == value){ + count++ + } + } + return count + } + + def findTotalPrice(def datas, def value){ + def count = 0 + def totalPrice = 0 + for(def data : datas){ + def type = data['T'] + def price = data['R'] + if(type == value){ + count++ + totalPrice = totalPrice + price + } + } + + def map = [:] + map.totalPrice = totalPrice + if(count){ + map.avgPrice = (totalPrice/count).toInteger() + } + return map + } + + def matchStat(){ + def citys = ["ÉϺ£","±±¾©","³É¶¼","ÖØÇì","ËÕÖÝ","¹ãÖÝ","´óÁ¬","ÏÃÃÅ","º¼ÖÝ","ÉîÛÚ"] + def datas = [] + + datas << ["³ÇÊÐ","ÇøÓò","Ãû³Æ"] + for(def city : citys){ + def rets = findLogs5(city); + def keys = [] + def priceMap = [:] + def cityDatas = [] + for(def ret : rets){ + if(ret.size()>=14 && ret[14] && ret[15] && ret[16]){ + def key = ret[14]+"#"+ret[15]+"#"+ret[16] +// def prices = priceMap.get(key) +// if(!prices){ +// prices = [] +// priceMap.put(key, prices) +// } +// prices << ret[18] +// if(!keys.contains(key)){ +// keys << key + cityDatas << [ret[14], ret[15], ret[16], ret[12],ret[18]] +// } + } +// if(ret.size()==14 && ret[13]=="ÔÝʱ²»Ö§³Ö¸ÃÐ¡ÇøµÄ¹À¼Û"){ +// def key = ret[0]+"#"+ret[1]+"#"+ret[2] +// if(!keys.contains(key)){ +// keys << key +// datas<<[ret[0], ret[1], ret[2], ret[12]] +// } +// } + + } + +// for(def k : keys){ +// def arrs = k.split("#") +// datas<<[arrs[0], arrs[1], arrs[2]] +// } + + +// for(def o : cityDatas){ +// def prices = priceMap.get(o[0]+"#"+o[1]+"#"+o[2]); +// def avg_price = prices.sum()/prices.size() +// o << avg_price.toInteger() +// } +// + datas.addAll(cityDatas) + +// def data = [] +// data << city +// data << keys.size() +// datas << data + } + ExcelUtils.write("D:\\Êý¾Ý\\ÉîÛÚ½¨ÐÐ\\½¨ÐÐÆ¥ÅäÐ¡Çø£¨2£©.xlsx",datas,"ͳ¼Æ") + } + + def statNoHouseCalType(){ + def citys = ["ÉϺ£","±±¾©","³É¶¼","ÖØÇì","ËÕÖÝ","¹ãÖÝ","´óÁ¬","ÏÃÃÅ","º¼ÖÝ","ÉîÛÚ"] + def results = [] + results << ["³ÇÊÐ","ÎÞÑù±¾×ÜÁ¿","1","2","3","4","5","6","7","8","9","10","ÆäËü"] + for(def city : citys){ + def rets = findLogs6(city); + println rets.size() + + def map = [:] + for(def r : rets){ + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(r[6])) + def obj = CommonDatautils.queryDistrictPrice(r[0], r[1], r[2], weekend) + if(obj){ + def type = map.get(obj.value_type) + if(!type){ + type = 0 + } + map.put(obj.value_type, type+1); + } + } + println city +" "+rets.size() + map.size() + + def rows = [] + rows << city + rows << rets.size() + for(def i=1;i<=10;i++){ + if(map.containsKey(i)){ + rows << map.get(i) + }else{ + rows << "" + } + } + results << rows + } + ExcelUtils.write("D:\\Êý¾Ý\\ÉîÛÚ½¨ÐÐ\\ÎÞÑù±¾¼ÆË㷽ʽͳ¼Æ.xlsx",results, "Êý¾Ý") + } + + def statCityHouse(){ + def citys = ["ÉϺ£","±±¾©","³É¶¼","ÖØÇì","ËÕÖÝ","¹ãÖÝ","´óÁ¬","ÏÃÃÅ","º¼ÖÝ","ÉîÛÚ"] + def rets = [] + rets << ["³ÇÊÐ","×ÜÁ¿","anjuke.com","58.com","ke.com","leju.com","lianjia.com","c21.com","fang.com","centanet.com","fangdd.com","qfang.com","fccs.com","house365.com","zhuge.com","izhiliao.com","leyoujia.com","5i5j.com","¾ù¼Û"] + def roots = ["anjuke.com","58.com","ke.com","leju.com","lianjia.com","c21.com","fang.com","centanet.com","fangdd.com","qfang.com","fccs.com","house365.com","zhuge.com","izhiliao.com","leyoujia.com","5i5j.com"] + for(def city : citys){ + def ret = [] + ret << city + def total = findHouseBySource(city, null); + ret << total + for(def root : roots){ + def count = findHouseBySource(city, root); + + def pre = DataHandleUtils.getDoubleRound(2, count.toInteger()*100/total.toInteger()) + ret << count+"(${pre}%)" + } + def cityPrice = CommonDatautils.findCityPrice(city, "2021-01-31") + ret << cityPrice + rets << ret + println ret + } + ExcelUtils.write("D:\\Êý¾Ý\\ÉîÛÚ½¨ÐÐ\\½¨Ðз¿Ô´Í³¼Æ.xlsx",rets,"ͳ¼Æ") + + } + + + + def findHouseBySource(def city, def root){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >=20200101") + if(root){ + sqlBuffer.append(" and root = '"+root+"'") + } + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def count = 0 + while (rs.next()){ + count = rs.getString("count") + } + return count; + } + + + def findTestDatas(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def count = 0 + println "×ÜÁ¿£º"+cursor.count() + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + if(resMap.calcType==value){ + + } + } + } + return count; + } + + def statNoHouseCCBData(){ + def citys = ["ÉϺ£","±±¾©","³É¶¼","ÖØÇì","ËÕÖÝ","¹ãÖÝ","´óÁ¬","ÏÃÃÅ","º¼ÖÝ","ÉîÛÚ"] + for(def city : citys){ + findTestDatas(city, 2) + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy new file mode 100644 index 000000000..7c630dfa9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy @@ -0,0 +1,117 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +class LogStat { + + static main(args) { + def task = new LogStat() + task.run() + } + + def run(){ + def datas = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐ\\½¨ÐÐÆ¥ÅäÐ¡Çø2.xlsx", 0, 1) + println datas.size() + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\½¨ÐÐÆ¥ÅäÐ¡Çø2.xlsx",{ d -> + def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd") + def weekend = DateUtils.getFirstDayOfWeek(calDate) + def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend) + if(dbo){ + d['E'] = dbo.avg_price + } + + def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(district){ + def startDate = DateUtil.addMonth(calDate, -6) + def start = startDate.replace("-","").toInteger() + def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger() + def alias = [] + if(district.alias){ + if(!district.alias.contains(district.name)){ + district.alias << district.name + } + alias = district.alias + }else{ + alias << district.name + } + def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end) + d['G'] = count + } + },0) + } + + + def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ + + String sql = "select * from seawater-20* where cat=sell and city='${city}' and region='${region}' and district_name in (${alias.join(',')})" + def rets = [] + ElasticUtils.executeSqlQuery(sql,{data-> + rets << data + println data + }) + return rets.size() + } + +// def findTestDatas(def city, def value) { +// def q = new BasicDBObject() +// q.put("user_id", "1007751") +// q.put("parameter_map.city", city) +// q.put("request_uri", "/property/evaluateForTest") +// +// def keys = new BasicDBObject() +// keys.put("res_map",1) +// def cursor = LOG.find(q) +// def count = 0 +// println "×ÜÁ¿£º"+cursor.count() +// def datas = [] +// for(def d : cursor){ +// if(d && d.code==200 && d.res_map && d.res_map.code==200){ +// def resMap = d.res_map.result +// if(resMap.calcType==value){ +// def data = [] +// def rowMap = [:] +// rowMap.city = resMap.city +// rowMap.region = resMap.district +// rowMap.name = resMap.name +// rowMap.calDate = d.parameter_map.calDate +// data << rowMap +// } +// } +// } +// return datas; +// } + +// for(def i=0;i3000){ + info1 = null + } + }else{ + info1 = null + } + } + + + if(midSchools) { + info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools); + if(info2 && info2.school_lng2){ + double lat = dist.lat/10000000000.0 + double lng = dist.lng/10000000000.0 + distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2) + if(distance2>3000){ + info2 = null + } + }else{ + info2 = null + } + } + if(info1|| info2){ + def row = [] + row << cityCodeMap.get(dist.city) + row << regionCodeMap.get(dist.city +"-"+dist.region) + row << dist.name + row << dist.lng/10000000000.0 + row << dist.lat/10000000000.0 + row << dist.address + if(info1){ + row << info1.school_name + row << info1.school_address!=null?info1.school_address:'' + row << info1.attribute!=null?info1.attribute:'' + row << info1.school_lng2!=null?info1.school_lng2:'' + row << info1.school_lat2!=null?info1.school_lat2:'' +// row << distance1 + }else{ + row << "" + row << "" + row << "" + row << "" + row << "" +// row << "" + } + + if(info2){ + row << info2.school_name + row << info2.school_address!=null?info2.school_address:'' + row << info2.attribute!=null?info2.attribute:'' + row << info2.school_lng2!=null?info2.school_lng2:'' + row << info2.school_lat2!=null?info2.school_lat2:'' +// row << distance2 + }else{ + row << "" + row << "" + row << "" + row << "" + row << "" +// row << "" + } + row << "" + row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss") + row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") + row << "fjw" + row << dist._id.toString() + row << "¶Ô¿Ú" + + rets << row + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + ExcelUtils.write("d:\\Êý¾Ý\\Â¥Å̶ԿÚѧУ_V2.xlsx", rets, "Êý¾Ý") + + } + + + def selectSchool(lng, lat, def schools){ + def minSchool = null + def level0 = []//¹ú¼Ò + def level1 = []//Ê¡ÖØµã + def level2 = []//ÊÐÖØµã + def level3 = []//ÇøÖØµã + def level4 = []//ÆÕͨ + def level5 = []//ÆäËü + for(def s : schools){ + if(s.attribute){ + if(s.attribute.contains("¹ú¼ÒÖØµã")){ + level0 << s + }else if(s.attribute.contains("Ê¡ÖØµã")){ + level1 << s + }else if(s.attribute.contains("ÊÐÖØµã")){ + level2 << s + }else if(s.attribute.contains("ÇøÖØµã")){ + level3 << s + }else if(s.attribute.contains("ÆÕͨ")){ + level4 << s + }else{ + level5 << s + } + }else{ + level5 << s + } + } + + + if(level0){ + if(level0.size()==1){ + minSchool = level0.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level0) + } + }else if(level1){ + if(level1.size()==1){ + minSchool = level1.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level1) + } + }else if(level2){ + if(level2.size()==1){ + minSchool = level2.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level2) + } + }else if(level3){ + if(level3.size()==1){ + minSchool = level3.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level3) + } + }else if(level4){ + if(level4.size()==1){ + minSchool = level4.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level4) + } + }else if(level5){ + if(level5.size()==1){ + minSchool = level5.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level5) + } + } + + def rets = [] + + if(minSchool){ + rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name) + } + + if(rets.size()>0){ + for(def d : rets){ + if(!minSchool.school_address){ + minSchool.school_address = d.address + } + if(!minSchool.attribute){ + minSchool.attribute = d.attribute + } + } + }else{ + if(minSchool){ + if(minSchool.school_map_lng){ + minSchool.school_lng2 = minSchool.school_map_lng + minSchool.school_lat2 = minSchool.school_map_lat + }else if(minSchool.school_lng1){ + minSchool.school_lng2 = minSchool.school_lng1 + minSchool.school_lat2 = minSchool.school_lat1 + }else if(minSchool.school_lng){ + minSchool.school_lng2 = minSchool.school_lng + minSchool.school_lat2 = minSchool.school_lat + } + } + + } + return minSchool; + } + + + def findSchoolInfo(def city, def region, def schoolName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", schoolName) + def dbcursor = school_col.find(q) + def rets = [] + for(def obj : dbcursor){ + rets << obj + } + return rets + } + + def selectMinDistanceSchool(def lat, def lng, def schools){ + def minSchool = null; + def minDistance = Integer.MAX_VALUE + for(def s1 : schools){ + + def lng2 = null + def lat2 = null + if(s1.school_map_lng){ + lng2 = s1.school_map_lng + lat2 = s1.school_map_lat + }else if(s1.school_lng1){ + lng2 = s1.school_lng1 + lat2 = s1.school_lat1 + }else if(s1.school_lng){ + lng2 = s1.school_lng + lat2 = s1.school_lat + } + if(lat2){ + def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2) + println s1.district_name +" "+ s1.school_name+" "+distance + if(distance + total = updateLocation(v, total); + println "---------------------------------------------------------------------------------------------------------------" + } + } + + def updateLocation(def lists, def total){ + + + for(def dist : lists){ + + def addrDistMap = getLocation(dist, dist.address) + def nameDistMap = getLocation(dist, dist.region + dist.name) + + if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){ + println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"") + // println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})" + + if(addrDistMap){ + def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|",""); + def nameKey = deleteRegion(nameDistMap.keyword, dist.region); + + def baidu = null + def lat = null + def lng = null + def upldateFlag = "" + if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("·") || addrKey.contains("´óµÀ")) && addrKey.contains("ºÅ") && dist.addr == addrKey){ + lat = addrDistMap.lat + lng = addrDistMap.lng + baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + upldateFlag = "Address" + }else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){ + lng = nameDistMap.lng + lat = nameDistMap.lat + baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + upldateFlag = "Name" + } + if(baidu){ + total++ + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + Long baiduLat = baidu.y*10000000000L + Long baiduLng = baidu.x*10000000000L + udata.put("lat2", baiduLat) + udata.put("lng2", baiduLng) + Long lat0 = lat*10000000000L + Long lng0 = lng*10000000000L + udata.put("lat", lat0) + udata.put("lng", lng0 ) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag + } + + } + } + } + return total + } + + def getMoreSameLocationDist(def districts){ + def locationMap = [:] + for(def i=0;i + if(v.size()>=2){ + + v.sort{ a,b-> + return a.name.size().compareTo(b.name.size()) + } + resMap.put(k,v) + } + } + + def lastRetMaps = [:] + resMap.each{k,v-> + def swname = null + def count = 0 + for(def i=0;i-1){ + if(s.indexOf(region+"Çø")>-1){ + key = s.substring(s.indexOf(region+"Çø")+region.size()+1, s.size()); + }else if(s.indexOf(region+"ÏØ")>-1){ + key = s.substring(s.indexOf(region+"ÏØ")+region.size()+1, s.size()); + }else if(s.indexOf(region+"ÊÐ")>-1){ + key = s.substring(s.indexOf(region+"ÊÐ")+region.size()+1, s.size()); + } + }else{ + key = s; + } + return key + } + + + + def getLocation(dist, addr){ + + if(addr){ + def rMap = [:] + def reps = GaoDeApiUtils.geo(dist.city, addr); + if(reps && reps.count.toInteger()>=1){ + def map = reps.geocodes.get(0) + def location = map.location + def level = map.level + if(location){ + def locMap = location.split(",") + def lat = dist.lat/10000000000.0; + def lng = dist.lng/10000000000.0; + def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng) + rMap.distance = distance + rMap.level = level + rMap.lat = locMap[1].toDouble() + rMap.lng = locMap[0].toDouble() + rMap.keyword = map.formatted_address + rMap.location = location + rMap.region = map.district + return rMap; + } + } + } + return null; + } + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FzLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FzLocationBatch.groovy new file mode 100644 index 000000000..4345374e2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FzLocationBatch.groovy @@ -0,0 +1,4 @@ +package com.fangjia.script.gj.test + +class FzLocationBatch { +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy index cc5c8b7b3..838fcbd20 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy @@ -30,7 +30,9 @@ class JiangShuBlankDataStat { // task.matchHouse(); // task.villa() - task.addressParse() +// task.addressParse() + + task.deleteProcess() } def repeat(){ @@ -288,5 +290,22 @@ class JiangShuBlankDataStat { return floor; } + def deleteProcess(){ + + def datas = ExcelUtils.read("D:\\Êý¾Ý\\½­ËÕÅ©ÐÐ\\½­ËÕ2ǧ.xlsx",0,1) + def addrList = [] + for(def d : datas){ + def addr = d['G'] + addrList << addr + } + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½­ËÕÅ©ÐÐ\\200ƽÒÔÏÂ.xlsx", { row -> + def addr = row['C'] + if(addrList.contains(addr)){ + row['R'] = "Òѵ÷²é" + } + + },0); + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy new file mode 100644 index 000000000..1a1f3fb66 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -0,0 +1,3 @@ +package com.fangjia.script.gj.test + +println "wwww" \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index cc38a3d50..15910f5cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐÔ­µØ²ú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐÔ­µØ²ú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿"] for(def source :sources){ - def count1 = findTotalCount(source, 202101); - def count2 = findTotalCount(source, 202102); + def count1 = findTotalCount(source, 202102); + def count2 = findTotalCount(source, 202103); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AddFangSchoolRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AddFangSchoolRegion.groovy new file mode 100644 index 000000000..98646b77d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AddFangSchoolRegion.groovy @@ -0,0 +1,213 @@ +package com.fangjia.script.thirdPriceCrawl.other + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import net.sf.json.groovy.JsonSlurper + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class AddFangSchoolRegion { + + static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school") + static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price") + static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new AddFangSchoolRegion(); + //task.process() + //task.match() + task.addMapLocation() + } + + def addMapLocation(){ + def q = new BasicDBObject() + def dbcursor = district_school_col.find(q) + def index = 0 + for(def d : dbcursor){ + def data = findSchool(d.city, d.region, d.school_name) + if(data && data.location){ + def location = data.location + if(location){ + def udata = new BasicDBObject() + udata.put("school_map_lng", location[0]) + udata.put("school_map_lat", location[1]) + district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata)) + } + + } + index ++ + println index + } + } + + def match(){ + def datas = findAllDatas() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;itotalPage){ + stop = true; + } + schools.addAll(data.list) + } + } + } + } + + print(schools.size()); + + def districtSchools = [] + def schoolInfos = [] + if(schools.size()>0){ + + def url = "https://51yongka.com/fapi/school/details" + for(def d : schools){ + def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM" + def s = chaxuequInfoCrawl(url, params) + JSONObject jsonObject = JSONObject.fromObject(s); + if(jsonObject && jsonObject.code==200){ + + if(jsonObject.data && jsonObject.data.nearbyPropertyList){ + def districts = jsonObject.data.nearbyPropertyList + for(def dist : districts){ + def dataMap = new BasicDBObject() + dataMap.city = jsonObject.data.city + dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area) + /**ÀàÐÍ: Ó×¶ùÔ°,Сѧ,³õÖÐ*/ + dataMap.category = converCategory(jsonObject.data.schoolName) + dataMap.district_name = dist.name.trim() + dataMap.district_address = dist.address + dataMap.school_name = jsonObject.data.schoolName.trim() + dataMap.school_address = jsonObject.data.address + dataMap.source = "²éÑ§Çø·¿" + dataMap.url = url + "?"+params + dataMap.telephone = jsonObject.data.officialTelephone; + districtSchools << dataMap + } + }else{ + + def schoolData = new BasicDBObject() + schoolData.province = "½­ËÕ" + schoolData.city = jsonObject.data.city + schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area) + /**ÀàÐÍ: Ó×¶ùÔ°,Сѧ,³õÖÐ*/ + schoolData.category = converCategory(jsonObject.data.schoolName) + schoolData.name = jsonObject.data.schoolName.trim() + schoolData.address = jsonObject.data.address + /**À´Ô´ÍøÕ¾*/ + schoolData.source = "²éÑ§Çø·¿" + /**·¿Ô´ÏêÇéurl*/ + schoolData.telephone = jsonObject.data.officialTelephone; + schoolInfos << schoolData + println jsonObject.data.schoolName +" ÎÞÖܱßÂ¥ÅÌ" + } + + } + println s; + } + } + + println "Ð¡Çø£º"+districtSchools.size() +// saveChaxuequDistrictSchool(districtSchools) + saveChaxuequSchool(schoolInfos) + + } + + def saveChaxuequSchool(schoolInfos){ + for(def ds : schoolInfos){ + def obj = findSchoolData(ds.city, ds.school_name); + if(!obj){ + school_col.save(ds) + } + } + } + + + + + def saveChaxuequDistrictSchool(def districtSchools){ + for(def ds : districtSchools){ + def obj = findDistrictSchoolData(ds.city, ds.school_name); + if(!obj){ + district_school_col.save(ds) + } + } + } + + def findSchoolData(city, school_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("school_name", school_name) + return school_col.findOne(q) + } + + + def findDistrictSchoolData(city, school_name, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("school_name", school_name) + q.put("district_name", district_name) + return district_school_col.findOne(q) + } + + def converCategory(def schoolName){ + + if(schoolName){ + if(schoolName.contains("³õÖв¿")||schoolName.contains("ÖÐѧ")){ + return "³õÖÐ" + }else if(schoolName.contains("Сѧ")){ + return "Сѧ" + } + } + return null + } + + def static chaxuequInfoCrawl(url, bodyquery){ + def responseMap = [:] + try { + //headersÐÅÏ¢ + Map headers = new LinkedHashMap() + headers.put("Host","51yongka.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","75") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("content-type","application/x-www-form-urlencoded") + headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html") + headers.put("Accept-Encoding","gzip, deflate, br") + String s = doPost(url, bodyquery, headers) + return s + } catch (Exception e) { + e.printStackTrace() + } + return null + } + + + def static schoolList(page) { + def responseMap = [:] + String s = null + try { + //headersÐÅÏ¢ + Map headers = new LinkedHashMap() + headers.put("Host","m.gusuwang.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","40") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("content-type","application/x-www-form-urlencoded") + headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html") +// headers.put("Accept-Encoding","gzip, deflate, br") + + //paramsÐÅÏ¢ + Map params = new HashMap(); + params.put("secCode","198a53da6baa3aa6a5b99959a006be37"); +// //postÇëÇó + def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool" + s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers) + } catch (Exception e) { + e.printStackTrace() + } + return s + } + + + + + + + static String doPost(String strUrl, String content, Map headers) { + String result = ""; + OutputStream outputStream = null; + InputStream inputStream = null; + StringBuilder sb = new StringBuilder(); + try { + URL url = new URL(strUrl); + //ͨ¹ýµ÷ÓÃurl.openConnection()À´»ñµÃÒ»¸öеÄURLConnection¶ÔÏ󣬲¢ÇÒ½«Æä½á¹ûÇ¿ÖÆ×ª»»ÎªHttpURLConnection. + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection.setRequestMethod("POST"); + //ÉèÖÃÁ¬½ÓµÄ³¬Ê±ÖµÎª30000ºÁÃ룬³¬Ê±½«Å׳öSocketTimeoutExceptionÒì³£ + urlConnection.setConnectTimeout(30000); + //ÉèÖöÁÈ¡µÄ³¬Ê±ÖµÎª30000ºÁÃ룬³¬Ê±½«Å׳öSocketTimeoutExceptionÒì³£ + urlConnection.setReadTimeout(30000); + //½«urlÁ¬½ÓÓÃÓÚÊä³ö£¬ÕâÑù²ÅÄÜʹÓÃgetOutputStream()¡£getOutputStream()·µ»ØµÄÊä³öÁ÷ÓÃÓÚ´«ÊäÊý¾Ý + urlConnection.setDoOutput(true); + urlConnection.setDoInput(true); + //ÉèÖÃͨÓÃÇëÇóÊôÐÔΪĬÈÏä¯ÀÀÆ÷±àÂëÀàÐÍ +// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded"); + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + urlConnection.setRequestProperty(head, headers.get(head)); + } + } + //getOutputStream()·µ»ØµÄÊä³öÁ÷£¬ÓÃÓÚдÈë²ÎÊýÊý¾Ý¡£ + outputStream = urlConnection.getOutputStream(); + outputStream.write(content.getBytes()); + outputStream.flush(); + outputStream.close(); + //´Ëʱ½«µ÷Óýӿڷ½·¨¡£getInputStream()·µ»ØµÄÊäÈëÁ÷¿ÉÒÔ¶ÁÈ¡·µ»ØµÄÊý¾Ý¡£ + inputStream = urlConnection.getInputStream(); + + +// byte[] data = new byte[1024]; +// //inputStreamÿ´Î¾Í»á½«¶ÁÈ¡1024¸öbyteµ½dataÖУ¬µ±inputSteamÖÐûÓÐÊý¾Ýʱ£¬inputStream.read(data)ֵΪ-1 +// while (inputStream.read(data) != -1) { +// String s = new String(data, Charset.forName("utf-8")); +// sb.append(s); +// } + + InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8"); + char[] buff = new char[1024]; + int length = 0; + while ((length = reader.read(buff)) != -1) { + String s = new String(buff, 0, length); + sb.append(s); + } + } catch (IOException e) { + e.printStackTrace(); + }finally{ + if(outputStream!=null){ + outputStream.close(); + } + if(inputStream!=null){ + inputStream.close(); + } + } + + return sb.toString(); + } + + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 408763cfd..3f43734e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -52,7 +52,7 @@ class ElasticUtils { return dataSource; } - static executeSqlQuery(String sql, Closure closure, String host = "192.168.0.216"){ + static executeSqlQuery(String sql, Closure closure, String host = "192.168.0.206"){ EsDataSource dataSource = getEsDataSource(host) Connection connection = dataSource.getConnection() Statement statement = connection.createStatement() diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java index 6015a13ea..2ffb4363c 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java @@ -7,14 +7,14 @@ import java.util.regex.Pattern; */ public interface AddressParserPatterns { - Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(·|½Ö|´óµÀ|Ïï|µÀ|((Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±|¸´)[Ïß»·Ìõ¼×])|(ºúͬ))(½»»ã|.{0,3}¶Î|.*Ū)?((([¼×ÒÒ]?[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+¼×?ºÅ?)(?!Â¥)[~\\-¡«¡¢,£­])?([¼×ÒÒ]?[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d])+¼×?(ºÅ)(?!Â¥)+([¸¶¸½]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?)?)"); + Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(·|½Ö|´óµÀ|Ïï|µÀ|((Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±|¸´)[Ïß»·Ìõ¼×])|(ºúͬ))(½»»ã|.{0,3}¶Î|.*Ū)?((([¼×ÒÒ]?[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+¼×?ºÅ?)(?!Â¥)[~\\-¡«¡¢,£­])?([¼×ÒÒ]?[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d])+¼×?(ºÅ)(?!Â¥)+([¸¶¸½]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?+([-]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?)?)?)"); Pattern STREET = Pattern.compile("^.{2,}(·|½Ö|´óµÀ|Ïï|µÀ|((Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±|¸´)[Ïß»·Ìõ¼×])|(ºúͬ))(½»»ã|.{0,3}¶Î|.*Ū)?"); - Pattern STREET_NUMBER = Pattern.compile(".*([¼×ÒÒ]?[Áã|Ò»ÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+¼×?(ºÅ)(?!Â¥)?[~\\-£­¡«¡¢,])?([¼×ÒÒ]?[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+¼×?(ºÅ)(?!Â¥)+([¸¶¸½]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?)?)"); + Pattern STREET_NUMBER = Pattern.compile(".*([¼×ÒÒ]?[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+¼×?(ºÅ)(?!Â¥)?[~\\-£­¡«¡¢,])?([¼×ÒÒ]?[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+¼×?(ºÅ)(?!Â¥)+([¸¶¸½]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?+([-]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?)?)?)"); Pattern STAGE_PATTERN = Pattern.compile("(([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+)ÆÚ?[£¬¡¢ÖÁµ½]?)?([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+)ÆÚ"); Pattern PARTITION_PATTERN = Pattern.compile("(\\w|¶«|ÄÏ|±±|ÖÐ|Î÷|¼×|ÒÒ|±û|¶¡|Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®)+(Çø|·»)"); Pattern TOWN = Pattern.compile("^.+?(Ïç|Õò|½ÖµÀ)+"); Pattern NUMBER = Pattern.compile("\\d+"); - Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(·|½Ö|´óµÀ|Ïï|µÀ|((Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±|¸´)[Ïß»·Ìõ¼×])|(ºúͬ))(½»»ã|.{0,3}¶Î|.*Ū)?((([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d])+Ū)|([¼×ÒÒ]?([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]))+ºÅ(?!Â¥)+([¸¶¸½]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?)?)"); + Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(·|½Ö|´óµÀ|Ïï|µÀ|((Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±|¸´)[Ïß»·Ìõ¼×])|(ºúͬ))(½»»ã|.{0,3}¶Î|.*Ū)?((([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d])+Ū)|([¼×ÒÒ]?([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]))+ºÅ(?!Â¥)+([¸¶¸½]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?+([-]+[ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ºÅ)?)?)?)"); Pattern DISTRICT = Pattern.compile("^.{2,}?(Çø|ÏØ)"); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 47d32dc97..da76b0bc4 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,6 +1,7 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; +import com.fangjia.core.util.CNNMFilter; import com.fangjia.core.util.DataUtil; import com.fangjia.core.util.GeoUtils; import com.fangjia.core.util.StringUtil; @@ -17,6 +18,7 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors; import com.fangjia.service.address.processor.AddressProcessors; import com.fangjia.service.api.AliyunService; import com.fangjia.service.bo.AliyunBo; +import com.fangjia.service.bo.MatchAddressScore; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.addrepo.AddressRepoService; import com.fangjia.service.mongo.bkoff.HouseLoudongService; @@ -170,7 +172,9 @@ public class AddressParserServiceImpl implements AddressParserService { if(aliyunBo != null){ String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", ""); if(StringUtils.isNotBlank(poi)){ - addr.setKeywords(poi); + if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){ + addr.setKeywords(poi); + } } if(StringUtils.isNotBlank(aliyunBo.getCity())){ addr.setSuggestCity(aliyunBo.getCity()); @@ -189,6 +193,45 @@ public class AddressParserServiceImpl implements AddressParserService { if(params.isUseAliyunApi() && aliyunBo != null){ completBuildingField(matchedBuilding, aliyunBo, params); } + Address tempAddr = matchedBuilding.getSearchAddress(); + if(StringUtils.isBlank(tempAddr.getProvince()) || StringUtils.isBlank(tempAddr.getCity()) || StringUtils.isBlank(tempAddr.getDistrict())|| StringUtils.isBlank(tempAddr.getStreet())){ +// List metaDatas = addressRepoService.findAllByKeyword(null, params.getBuildingType(), new ArrayList() {{add(tempAddr.getOrgAddress());}}, tempAddr.getLng(), tempAddr.getLat(), 100, false); + tempAddr.addKeyword(tempAddr.getOrgAddress()); + List metaDatas = findMatchMetaData(tempAddr, params.getBuildingType(), MetaDataType.MAP, false); +// String addressStr = (StringUtils.isBlank(tempAddr.getStreet())?"":tempAddr.getStreet()) + (StringUtils.isBlank(tempAddr.getStreetNumber())?"":tempAddr.getStreetNumber()); + Map> metaDataMap = new HashMap>(); + if(StringUtils.isBlank(tempAddr.getDistrict()) && !metaDatas.isEmpty()){ + for(AddressMatchMetaData metaData:metaDatas){ + if(StringUtils.isNotBlank(metaData.getMapCity())){ + List metaList = metaDataMap.containsKey(metaData.getMapCity()) ? metaDataMap.get(metaData.getMapCity()) : new ArrayList(); + metaList.add(metaData); + metaDataMap.put(metaData.getMapCity(),metaList); + } + } + String tempCity = ""; + if(StringUtils.isNotBlank(tempAddr.getCity()) && metaDataMap.containsKey(tempAddr.getCity())){ + tempCity = tempAddr.getCity(); + }else{ + int size = 0; + for(String key:metaDataMap.keySet()){ + if(metaDataMap.get(key).size() > size){ + size = metaDataMap.get(key).size(); + tempCity = key; + } + } + } + if(StringUtils.isNotBlank(tempCity)){ + AddressMatchMetaData meta = metaDataMap.get(tempCity).get(0); + tempAddr.setProvince(meta.getMapProvince()); + tempAddr.setCity(meta.getMapCity()); + tempAddr.setDistrict(meta.getDistrict()); + if((tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){ + matchedBuilding.setAddress(meta.getAddress()); + } + } + } + } + matchedBuilding.setSearchAddress(tempAddr); return matchedBuilding; } @@ -224,12 +267,25 @@ public class AddressParserServiceImpl implements AddressParserService { if(StringUtils.isBlank(addr.getTownship())){ addr.setTownship(aliyunBo.getTown()); } - addr.setOrgBuildingNumber(aliyunBo.getHouseno()); + /*addr.setOrgBuildingNumber(aliyunBo.getHouseno()); addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); - addr.setUnitNumber(aliyunBo.getCellno()); - addr.setOrgRoomNumber(aliyunBo.getRoomno()); - addr.setRoomNumber(aliyunBo.getRoomno()); + addr.setUnitNumber(aliyunBo.getCellno());*/ + if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(²ã|Â¥)$")){ + addr.setRoomNumber(aliyunBo.getRoomno()); + addr.setRoomNumber(aliyunBo.getRoomno()); + }else{ + addr.setRoomNumber(null); + addr.setRoomNumber(null); + } + + /*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", ""); + String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", ""); + if(orgRoom!=null && orgRoom.equals(aliRoom)){ + addr.setRoomNumber(aliyunBo.getRoomno()); + }else{ + addr.setRoomNumber(null); + }*/ if(StringUtils.isBlank(addr.getStreet())){ addr.setStreet(aliyunBo.getRoad()); @@ -362,7 +418,7 @@ public class AddressParserServiceImpl implements AddressParserService { * @param metaDataType * @return */ - private List findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType){ + private List findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType, boolean validCity){ if(MetaDataType.MAP == metaDataType){ List keywords = address.getKeywordList(); if(keywords.isEmpty()){ @@ -375,7 +431,7 @@ public class AddressParserServiceImpl implements AddressParserService { } keywords.add(standardAddress); } - return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100); + return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100, validCity); }else{ return SEAWEED_LOCAL_CACHE.get(new MetaDataCacheKey(address.getCity(), buildingType, MetaDataType.CPDB)); } @@ -517,20 +573,20 @@ public class AddressParserServiceImpl implements AddressParserService { scoreResult.setOrgAddress(addr.getOrgAddress()); if(buildingType != null) { if(MetaDataType.MAP == metaDataType){ - metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP); - metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB)); + metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP, true); + metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true)); }else{ - metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB); + metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true); } } else { if(MetaDataType.MAP == metaDataType){ - metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP); + metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP, true); } for (BuildingType bt : addr.guessBuildType()) { if(metaDatas == null){ - metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB); + metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB, true); }else{ - metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB)); + metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB, true)); } } } @@ -967,7 +1023,12 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getStreetNumber()); } if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ - formattedAddressBuilder.append(building.getAddress()); + Address orgAddr = new Address(); + orgAddr.setOrgAddress(building.getAddress()); + Address addr = AddressProcessors.process(orgAddr); + building.getSearchAddress().setStreet(addr.getStreet()); + building.getSearchAddress().setStreetNumber(addr.getStreetNumber()); + formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber())); } if(StringUtils.isNotBlank(building.getSuggestName())){ formattedAddressBuilder.append(building.getSuggestName()); @@ -985,16 +1046,16 @@ public class AddressParserServiceImpl implements AddressParserService { && address.getStreetNumber().endsWith(address.getOrgBuildingNumber())){ }else{ - formattedAddressBuilder.append(address.getBuildingNumber()); -// formattedAddressBuilder.append(address.getBuildingNumber() + "¶°"); +// formattedAddressBuilder.append(address.getBuildingNumber()); + formattedAddressBuilder.append(address.getBuildingNumber() + "¶°"); } } if(StringUtils.isNotBlank(address.getOrgUnitNumber())){ - formattedAddressBuilder.append(address.getUnitNumber()); -// formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª"); +// formattedAddressBuilder.append(address.getUnitNumber()); + formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª"); } - if(StringUtils.isNotBlank(address.getOrgRoomNumber())){ + if(StringUtils.isNotBlank(address.getRoomNumber())){ formattedAddressBuilder.append(address.getRoomNumber()); // formattedAddressBuilder.append(address.getRoomNumber() + "ÊÒ"); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java index cc1334533..5f61f7dfa 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java @@ -259,7 +259,38 @@ public class AddressFormatterFactory { @Override protected String doFormat(String address) { String matchStr = DataUtil.match(address, TARGET); - String formatStr = matchStr + "ºÅ"; + String formatStr = matchStr.replaceAll("¶Î-","¶Î"); + if(!matchStr.endsWith("ºÅ")){ + formatStr += "ºÅ"; + } + return address.replace(matchStr,formatStr); + } + }); + + /** + * ½£ÄÏ·¶«¶Î-12ºÅ-18ºÅ + */ + formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(·|½Ö|´óµÀ|Ïï|µÀ|((Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±|¸´)[Ïß»·Ìõ¼×])|(ºúͬ))(½»»ã|.{0,3}¶Î|.*Ū)?((([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d])+Ū)|([¼×ÒÒ-]?\\d+[ºÅ](¸½\\d+(ºÅ)?(-\\d+(ºÅ)?)?)?([-][¸½¸¶¡¢]?\\d+(ºÅ(¸½½ü)?)?)?))")) { + final Pattern TARGET = Pattern.compile("(·|½Ö|´óµÀ|Ïï|µÀ|((Áã|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±|¸´)[Ïß»·Ìõ¼×])|(ºúͬ))(½»»ã|.{0,3}¶Î|.*Ū)?((([ÁãÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d])+Ū)|([¼×ÒÒ-]?\\d+[ºÅ](¸½\\d+(ºÅ)?(-\\d+(ºÅ)?)?)?([-][¸½¸¶¡¢]?\\d+(ºÅ(¸½½ü)?)?)?))"); + @Override + protected String doFormat(String address) { + String matchStr = DataUtil.match(address, TARGET); + String formatStr = matchStr; + if(!matchStr.endsWith("ºÅ")){ + formatStr += "ºÅ"; + } + return address.replace(matchStr,formatStr); + } + }); +/** + * ½£ÄÏ·¶«¶Î-12ºÅ-18ºÅ + */ + formatters.add(new AbstractAddressFormatter(Pattern.compile("(((·|½Ö|´óµÀ|Ïï|µÀ|¶Î|¸½|¸¶)-)|(-(¸½|¸¶)))")) { + final Pattern TARGET = Pattern.compile("(((·|½Ö|´óµÀ|Ïï|µÀ|¶Î|¸½|¸¶)-)|(-(¸½|¸¶)))"); + @Override + protected String doFormat(String address) { + String matchStr = DataUtil.match(address, TARGET); + String formatStr = matchStr.replace("-",""); return address.replace(matchStr,formatStr); } }); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java index 3fcee87e0..18e331f8b 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java @@ -44,6 +44,24 @@ public class AddressMatchMetaData { private Double lat; private String map; + public String getMapProvince() { + return mapProvince; + } + + public void setMapProvince(String mapProvince) { + this.mapProvince = mapProvince; + } + + public String getMapCity() { + return mapCity; + } + + public void setMapCity(String mapCity) { + this.mapCity = mapCity; + } + + private String mapProvince; + private String mapCity; public void addName(String name, float priority){ if(StringUtils.isNotBlank(name)){ String formatName = AddressParserUtil.standardAddressAndName(name); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 2dd886f8b..98fac4be7 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -50,18 +50,23 @@ public class CityProcessor extends AbstractAddressProcessor { return RepeatStatus.NEXT; } } - if(StringUtils.isNotBlank(city)){ + if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ String formatCity = RegionData.formatCity(city); - String possibleCity = null; + String possibleCity = parseCityFromAddress(orgAddress, address); + String confirmCity = null; /**Èç¹ûÊ¡·ÝºÍ³ÇÊÐÐÅÏ¢Ï໥ì¶Ü ÔòÓÃÇøÓòÍÆ¶Ï³ÇÊÐ*/ if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){ List possibleCities = speculateCityNameByRegion(address, city); if(isUnique(possibleCities)){ formatCity = getFirst(possibleCities); + confirmCity = formatCity; } } - if(null != formatCity){ - city = formatCity; + if(formatCity == null || !formatCity.equals(possibleCity)){ + confirmCity = possibleCity; + } + if(null != confirmCity){ + city = confirmCity; address.setCity(city); address.setCityStatus(CityStatus.SURE); } @@ -98,7 +103,7 @@ public class CityProcessor extends AbstractAddressProcessor { address.setCity(city); } if(StringUtils.isNotBlank(city) && scoreRegion(city, orgAddress) > 0.4){ - orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[ÊÐÏØÇøÖÝŽˆ]?" , ""); + orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[ÊÐÏØÇøÖÝŽˆ]?(?!(·|½Ö|´óµÀ|Ïï|µÀ|¶Î))" , ""); address.setAddress(orgAddress); address.setCityStatus(CityStatus.SURE); return RepeatStatus.NEXT; @@ -166,7 +171,7 @@ public class CityProcessor extends AbstractAddressProcessor { String street = getStreet(address); for(String city : allCity){ if(street != null){ - MatchResult matchResult = matchFirst(Pattern.compile(city + "[¶«ÄÏÎ÷±±ÖÐÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]?(·|½Ö|´óµÀ|֧·|´å)"), street); + MatchResult matchResult = matchFirst(Pattern.compile(city + "[¶«ÄÏÎ÷±±ÖÐÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]?(·|½Ö|´óµÀ|´ó½Ö|֧·|´å)"), street); if(matchResult !=null){ address = address.replace(matchResult.matchContent, ""); } @@ -199,6 +204,10 @@ public class CityProcessor extends AbstractAddressProcessor { } Map> regionMap = RegionData.getAllRegions(city); for(String region : regionMap.keySet()){ + MatchResult matchResult = matchFirst(Pattern.compile(region + "[¶«ÄÏÎ÷±±ÖÐÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]?(·|½Ö|´óµÀ|´ó½Ö|֧·|´å)"), street); + if(matchResult !=null){ + address = address.replace(matchResult.matchContent, ""); + } if(address.indexOf(region, index) > -1){ if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){ weight.score += 2; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java index 36c916555..0f9759d2e 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java @@ -24,9 +24,9 @@ public class ProvinceProcessor extends AbstractAddressProcessor { String orgAddress = address.getAddress(); String[] provinceInfo = getProvince(orgAddress); - if(provinceInfo == null && address.getCity() != null){ + /*if(provinceInfo == null && address.getCity() != null){ provinceInfo = getProvince(address.getCity()); - } + }*/ if (null != provinceInfo) { String province = provinceInfo[NAME_IDX]; if (provinceInfo[FULL_NAME_IDX] != null) {//È«³ÆÆ¥Åä diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java index 77d00e97d..92ad74b29 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java @@ -18,6 +18,7 @@ public class RoomNumberProcessor extends AbstractAddressProcessor { static final Pattern NUMBER_END = Pattern.compile("(?<=\\D)\\d{3,4}$"); static final Pattern HOUSE_NUMBER = Pattern.compile("\\d{3,4}"); static final String HOUSE_UNITS_PATTERN = "(ÊÒ|·¿|ºÅ·¿|ºÅ(?!Ôº)|»§|ÆÌ)+"; + static final Pattern HOUSE_PATTERN_CASE02 = Pattern.compile("(\\d+(ÊÒ|·¿|ºÅ)?[¡¢]?)+$"); @Override public RepeatStatus process(Address address) { @@ -27,10 +28,17 @@ public class RoomNumberProcessor extends AbstractAddressProcessor { orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end); address.setOrgRoomNumber(matchResult.matchContent); address.setAddress(orgAddress); - analyFloorUseRoomNumber(address); +// analyFloorUseRoomNumber(address); + return RepeatStatus.NEXT; + } + MatchResult matchResult2 = matchFirst(HOUSE_PATTERN_CASE02, orgAddress); + if (matchResult2 != null) { + orgAddress = removeSubstring(orgAddress, matchResult2.start, matchResult2.end); + address.setOrgRoomNumber(matchResult2.matchContent); + address.setAddress(orgAddress); +// analyFloorUseRoomNumber(address); return RepeatStatus.NEXT; } - matchResult = matchFirst(HOUSE_PATTERN, orgAddress); if (matchResult != null) { orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end); @@ -39,10 +47,10 @@ public class RoomNumberProcessor extends AbstractAddressProcessor { } else { address.setOrgRoomNumber(matchResult.matchContent + "ÊÒ"); } - analyFloorUseRoomNumber(address); +// analyFloorUseRoomNumber(address); address.setAddress(orgAddress); } - analyFloorUseRoomNumber(address); +// analyFloorUseRoomNumber(address); removeHouseUnit(address); return RepeatStatus.NEXT; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java index 976f2925f..4d3b65b14 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java @@ -41,6 +41,7 @@ public class TooltipProcessor extends AbstractAddressProcessor { offset += matchResult.matchContent.length(); if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){ address.setTooltip(tooltip); + address.setStreet(getStreet(tooltip)); } else if(containsDistrictName(tooltip)){ address.setTooltip(tooltip); } else if(RegionData.isRegion(address.getCity(), tooltip)){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java index 0d9295ce6..5236b7d55 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java @@ -1,6 +1,7 @@ package com.fangjia.service.address.processor; import com.fangjia.service.address.mode.Address; +import com.fangjia.service.cnAnalyzer.Token; import org.apache.commons.lang.StringUtils; import java.util.ArrayList; @@ -14,6 +15,15 @@ public class VillageNameProcessor extends AbstractAddressProcessor { @Override public AbstractAddressProcessor.RepeatStatus process(Address address) { String orgAddress = address.getAddress(); + if(address.getTokens()!=null){ + for(Token token:address.getTokens()){ + if("district".equals(token.getKeywordTypes())){ + address.setKeywords(token.getName()); + address.setAddress(orgAddress.replace(token.getName(),"")); + return RepeatStatus.NEXT; + } + } + } List names = tokenDistrictNames(orgAddress); List kws = new ArrayList(); for(String name : names){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java index 227aec149..9a5c6d1d8 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java @@ -68,7 +68,7 @@ public interface AmapPoiService { * @param types * @return */ - List search(String city, String keywords, Integer limit, String types); + List search(String city, String keywords, Integer limit, String types, boolean validCity); /** diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java index 6060696b5..91a08b863 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java @@ -1,5 +1,6 @@ package com.fangjia.service.api; +import com.fangjia.core.ProvinceData; import com.fangjia.core.RegionData; import com.fangjia.core.util.DataUtil; import com.fangjia.core.util.GeoUtils; @@ -258,14 +259,14 @@ public class AmapPoiServiceImpl implements AmapPoiService { } @Override - public List search(String city, String keywords, Integer limit, String types) { + public List search(String city, String keywords, Integer limit, String types, boolean validCity) { Map params = new HashMap(); String area = city; params.put("city", area); params.put("output", "json"); params.put("keywords", keywords); params.put("key", "4ccfe3c898a9489884bd2bcb1ef6dcde"); - params.put("citylimit", "true"); +// params.put("citylimit", "true"); params.put("offset", limit == null?"25":String.valueOf(limit)); if(types != null){ params.put("types", "120301&120302&120201&060101"); @@ -351,15 +352,17 @@ public class AmapPoiServiceImpl implements AmapPoiService { cityname = poi.getString("cityname"); } cityname = RegionData.formatCity(cityname); - if(!city.equals(cityname)){ + if((city!=null && !city.equals(cityname) && validCity) || cityname == null){ continue; } + pname = ProvinceData.formatProvince(pname); AmapPoi amapPoi = new AmapPoi(); - amapPoi.setCity(city); + amapPoi.setCity(cityname); + amapPoi.setPname(pname); if(poi.containsKey("adname") && poi.get("adname") instanceof String){ adname = poi.getString("adname"); - String region = RegionData.formatRegion(city, adname); - if(RegionData.isRegion(city, region)){ + String region = RegionData.formatRegion(cityname, adname); + if(RegionData.isRegion(cityname, region)){ amapPoi.setRegion(region); } } @@ -380,8 +383,8 @@ public class AmapPoiServiceImpl implements AmapPoiService { amapPoi.setAlias(alias); } if(poi.containsKey("adname") && poi.get("adname") instanceof String){ - String region = RegionData.formatRegion(city, poi.getString("adname")); - if(RegionData.isRegion(city, region)){ + String region = RegionData.formatRegion(cityname, poi.getString("adname")); + if(RegionData.isRegion(cityname, region)){ amapPoi.setRegion(region); } } @@ -391,7 +394,7 @@ public class AmapPoiServiceImpl implements AmapPoiService { .replace(cityname, "").replace(adname, ""); amapPoi.setAddress(AddressParserUtil.optimizeAddress(address)); } - amapPoi.setCity(city); + amapPoi.setCity(cityname); if(poi.containsKey("location") && poi.get("location") instanceof String){ String[] location = poi.getString("location").split(","); amapPoi.setLng(Double.valueOf(location[0])); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java index 9aee32e52..6289384e4 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java @@ -37,7 +37,7 @@ public interface AddressRepoService { * @param max * @return */ - List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max); + List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max, boolean validCity); /** * diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java index a751826c7..3965c9f00 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java @@ -87,7 +87,7 @@ public class AddressRepoServiceImpl implements AddressRepoService { return addrs; } - public List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max){ + public List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max, boolean validCity){ String types = null; String tag = null; String category = null; @@ -104,7 +104,11 @@ public class AddressRepoServiceImpl implements AddressRepoService { tag = "°Ù»õÉ̳¡"; category = "×ÛºÏÉ̳¡"; } - List pois = amapPoiService.search(city, StringUtils.join(keywords, "|"), 20, types); + List pois = new ArrayList(); + for(String keyword:keywords){ + List poiList = amapPoiService.search(city, keyword, 20, types, validCity); + pois.addAll(poiList); + } List matchMetaDatas = new ArrayList(pois.size()); for(AmapPoi poi: pois){ AddressMatchMetaData data = new AddressMatchMetaData(); @@ -121,6 +125,8 @@ public class AddressRepoServiceImpl implements AddressRepoService { data.setType(poi.getType()); data.setMap("¸ßµÂ"); data.setCity(city); + data.setMapCity(poi.getCity()); + data.setMapProvince(poi.getPname()); if(poi.getAlias() != null){ for(String alia : poi.getAlias()){ data.addName(alia, MatchText.ALIAS_PRIORITY); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java index edc15e855..4fafa6663 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java @@ -31,10 +31,15 @@ public interface SeawaterService { void deleteById(String id); + /** + * ÅúÁ¿É¾³ý£¬Ó¢ÎĶººÅ·Ö¸î£¬Èç1,2,3 + * @param ids + */ + void deleteByIds(String ids); + @Deprecated List findAllSeawater(String category, String city, String name, String region, Integer room, String root, String sort, Integer startDate, Integer endDate, int start, int limit, boolean order, String[] fields); @Deprecated int findAllSeawaterCount(String category, String city, String name, String region, Integer room, String root, Integer startDate, Integer endDate); - } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java index d7e31ca58..2dc484035 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java @@ -12,10 +12,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.TermQuery; import org.apache.solr.client.solrj.util.ClientUtils; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; +import java.util.*; public class SeawaterServiceImpl implements SeawaterService { private SeawaterDao seawaterDao; @@ -188,6 +185,12 @@ public class SeawaterServiceImpl implements SeawaterService { seawaterDao.deleteById(id); } + @Override + public void deleteByIds(String ids) { + String[] idArr = ids.split(","); + seawaterDao.deleteByIds(Arrays.asList(idArr)); + } + public void setSeawaterDao(SeawaterDao seawaterDao) { this.seawaterDao = seawaterDao; } diff --git a/projects/geluke-service/pom.xml b/projects/geluke-service/pom.xml index 5dc163edc..08acecd69 100644 --- a/projects/geluke-service/pom.xml +++ b/projects/geluke-service/pom.xml @@ -29,7 +29,7 @@ org.sothis sothis-core - 0.2.0-SNAPSHOT + 0.2.0 provided @@ -154,7 +154,7 @@ com.poseidon poseidon-message - 0.0.1-SNAPSHOT + 0.0.1 org.mongodb