Merge branch 'script_gj' into 'master'
Suggestion乡镇街道从map_poi中查,提升速度 See merge request !789
This commit is contained in:
+2
-2
@@ -28,8 +28,8 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()//删除第三方
|
||||
task.delDealByPe()
|
||||
task.deleteThirdPartyData()//删除第三方
|
||||
task.delDealByPe()//删除成交
|
||||
// task.delDealData()
|
||||
|
||||
// task.exportThirdParty()
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.CNNMFilter
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.village.BaiduCoordinateConverter
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import net.sf.json.JSONObject
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
@@ -389,49 +390,32 @@ class OpenApiUtils {
|
||||
for (def loc : json.content) {
|
||||
String name = formatName(address)
|
||||
if(loc.uid) {
|
||||
def bean = [:]
|
||||
JSONObject uidInfo = BaiDuApiUtils.getDetailByUid(loc.uid)
|
||||
if(uidInfo != null) {
|
||||
def baiduRegion = uidInfo.get("result").area
|
||||
def baiduCity = uidInfo.get("result").city
|
||||
rets = createData(rets, uidInfo, loc, region, name, address)
|
||||
}else{
|
||||
def baiduRegion = loc.area_name.replace(loc.city_name,"")
|
||||
def baiduCity = loc.city_name
|
||||
baiduCity = RegionData.formatCity(baiduCity)
|
||||
def newName = formatName(loc.name)
|
||||
if(baiduRegion.contains(region) && (name== newName || address==newName)){
|
||||
JSONObject result = uidInfo.get("result")
|
||||
JSONObject lo = result.get("location")
|
||||
Double lng2 = lo.get("lng")
|
||||
Double lat2 = lo.get("lat")
|
||||
bean.lng2 = lng2
|
||||
bean.lat2 = lat2
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
|
||||
if(score1 || score2){
|
||||
def bean = [:]
|
||||
bean.name = loc.name
|
||||
bean.address = loc.addr.replace("[]","")
|
||||
bean.city = baiduCity
|
||||
bean.region = baiduRegion
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
def lng = gmap.get("lng")
|
||||
def lat = gmap.get("lat")
|
||||
bean.lng = lng
|
||||
bean.lat = lat
|
||||
if(result.get("detail_info")){
|
||||
bean.estateType = result.get("detail_info").get("tag")
|
||||
def locArrs = BaiduCoordinateConverter.bdMercator2Latlng(loc.diPointX/100.0, loc.diPointY/100.0)
|
||||
if(locArrs){
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0], locArrs[1])
|
||||
bean.lng = gmap.get("lng")
|
||||
bean.lat = gmap.get("lat")
|
||||
}
|
||||
|
||||
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
|
||||
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
|
||||
}else{
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
|
||||
rets << bean;
|
||||
}
|
||||
|
||||
bean.estateType = loc.std_tag
|
||||
rets << bean;
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+loc.name+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
|
||||
if(score1 || score2){
|
||||
rets << bean;
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
|
||||
}else{
|
||||
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
|
||||
}
|
||||
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -444,6 +428,67 @@ class OpenApiUtils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def static createData(def rets, def uidInfo, def loc, region, name, address){
|
||||
def bean = [:]
|
||||
def baiduRegion = uidInfo.get("result").area
|
||||
def baiduCity = uidInfo.get("result").city
|
||||
baiduCity = RegionData.formatCity(baiduCity)
|
||||
def newName = formatName(loc.name)
|
||||
if(baiduRegion.contains(region) && (name== newName || address==newName)){
|
||||
JSONObject result = uidInfo.get("result")
|
||||
JSONObject lo = result.get("location")
|
||||
Double lng2 = lo.get("lng")
|
||||
Double lat2 = lo.get("lat")
|
||||
bean.lng2 = lng2
|
||||
bean.lat2 = lat2
|
||||
bean.name = loc.name
|
||||
bean.address = loc.addr.replace("[]","")
|
||||
bean.city = baiduCity
|
||||
bean.region = baiduRegion
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
def lng = gmap.get("lng")
|
||||
def lat = gmap.get("lat")
|
||||
bean.lng = lng
|
||||
bean.lat = lat
|
||||
if(result.get("detail_info")){
|
||||
bean.estateType = result.get("detail_info").get("tag")
|
||||
}
|
||||
|
||||
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
|
||||
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
|
||||
}else{
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
|
||||
rets << bean;
|
||||
}
|
||||
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
|
||||
if(score1 || score2){
|
||||
|
||||
JSONObject result = uidInfo.get("result")
|
||||
JSONObject lo = result.get("location")
|
||||
bean.lng2 = lo.get("lng")
|
||||
bean.lat2 = lo.get("lat")
|
||||
bean.name = loc.name
|
||||
bean.address = loc.addr.replace("[]","")
|
||||
bean.city = baiduCity
|
||||
bean.region = baiduRegion
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(bean.lng2, bean.lat2)
|
||||
bean.lng = gmap.get("lng")
|
||||
bean.lat = gmap.get("lat")
|
||||
if(result.get("detail_info")){
|
||||
bean.estateType = result.get("detail_info").get("tag")
|
||||
}
|
||||
rets << bean;
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
|
||||
}else{
|
||||
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def static formatName(String name) {
|
||||
if(name.indexOf("·") != -1) {
|
||||
name = name.replaceAll("·", "")
|
||||
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package com.fangjia.script.gj.village
|
||||
import java.util.Arrays;
|
||||
class BaiduCoordinateConverter {
|
||||
|
||||
static double[] array1 = [75, 60, 45, 30, 15, 0]
|
||||
static double[][] array2 = [
|
||||
[-0.0015702102444, 111320.7020616939, 1704480524535203.0, -10338987376042340.0, 26112667856603880.0, -35149669176653700.0, 26595700718403920.0, -10725012454188240.0, 1800819912950474.0, 82.5],
|
||||
[0.0008277824516172526, 111320.7020463578, 647795574.6671607, -4082003173.641316, 10774905663.51142, -15171875531.51559, 12053065338.62167, -5124939663.577472, 913311935.9512032, 67.5],
|
||||
[0.00337398766765, 111320.7020202162, 4481351.045890365, -23393751.19931662, 79682215.47186455, -115964993.2797253, 97236711.15602145, -43661946.33752821, 8477230.501135234, 52.5],
|
||||
[0.00220636496208, 111320.7020209128, 51751.86112841131, 3796837.749470245, 992013.7397791013, -1221952.21711287, 1340652.697009075, -620943.6990984312, 144416.9293806241, 37.5],
|
||||
[-0.0003441963504368392, 111320.7020576856, 278.2353980772752, 2485758.690035394, 6070.750963243378, 54821.18345352118, 9540.606633304236, -2710.55326746645, 1405.483844121726, 22.5],
|
||||
[-0.0003218135878613132, 111320.7020701615, 0.00369383431289, 823725.6402795718, 0.46104986909093, 2351.343141331292, 1.58060784298199, 8.77738589078284, 0.37238884252424, 7.45]
|
||||
]
|
||||
static double[] array3 = [12890594.86, 8362377.87, 5591021, 3481989.83, 1678043.12, 0];
|
||||
static double[][] array4 = [
|
||||
[1.410526172116255e-8, 0.00000898305509648872, -1.9939833816331, 200.9824383106796, -187.2403703815547, 91.6087516669843, -23.38765649603339, 2.57121317296198, -0.03801003308653, 17337981.2],
|
||||
[-7.435856389565537e-9, 0.000008983055097726239, -0.78625201886289, 96.32687599759846, -1.85204757529826, -59.36935905485877, 47.40033549296737, -16.50741931063887, 2.28786674699375, 10260144.86],
|
||||
[-3.030883460898826e-8, 0.00000898305509983578, 0.30071316287616, 59.74293618442277, 7.357984074871, -25.38371002664745, 13.45380521110908, -3.29883767235584, 0.32710905363475, 6856817.37],
|
||||
[-1.981981304930552e-8, 0.000008983055099779535, 0.03278182852591, 40.31678527705744, 0.65659298677277, -4.44255534477492, 0.85341911805263, 0.12923347998204, -0.04625736007561, 4482777.06],
|
||||
[3.09191371068437e-9, 0.000008983055096812155, 0.00006995724062, 23.10934304144901, -0.00023663490511, -0.6321817810242, -0.00663494467273, 0.03430082397953, -0.00466043876332, 2555164.4],
|
||||
[2.890871144776878e-9, 0.000008983055095805407, -3.068298e-8, 7.47137025468032, -0.00000353937994, -0.02145144861037, -0.00001234426596, 0.00010322952773, -0.00000323890364, 826088.5]
|
||||
];
|
||||
|
||||
public static double[] convertor(double x, double y, double[] param) {
|
||||
double t = param[0] + param[1] * Math.abs(x);
|
||||
double cc = Math.abs(y) / param[9];
|
||||
double cf = param[2] + param[3] * cc + param[4] * cc * cc + param[5] * cc * cc * cc + param[6] * cc * cc * cc * cc + param[7] * cc * cc * cc * cc * cc + param[8] * cc * cc * cc * cc * cc * cc;
|
||||
t *= (x < 0) ? -1 : 1;
|
||||
cf *= (y < 0) ? -1 : 1;
|
||||
return [t, cf];
|
||||
}
|
||||
|
||||
public static double[] bdLatlng2Mercator(double lat, double lng) {
|
||||
double[] arr = null;
|
||||
double nLat = (lat > 74) ? 74 : lat;
|
||||
nLat = (nLat < -74) ? -74 : nLat;
|
||||
for (int i = 0; i < array1.length; i++) {
|
||||
if (lat >= array1[i]) {
|
||||
arr = array2[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (arr == null) {
|
||||
for (int i = array1.length - 1; i >= 0; i--) {
|
||||
if (lat <= -array1[i]) {
|
||||
arr = array2[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (arr == null) {
|
||||
throw new IllegalArgumentException("Invalid latitude");
|
||||
}
|
||||
double[] res = convertor(lng, lat, arr);
|
||||
return [res[0], res[1]]
|
||||
}
|
||||
|
||||
public static double[] bdMercator2Latlng(double x, double y) {
|
||||
double[] arr = null;
|
||||
double npX = Math.abs(x);
|
||||
double npY = Math.abs(y);
|
||||
for (int i = 0; i < array3.length; i++) {
|
||||
if (npY >= array3[i]) {
|
||||
arr = array4[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (arr == null) {
|
||||
throw new IllegalArgumentException("Invalid y value");
|
||||
}
|
||||
double[] res = convertor(npX, npY, arr);
|
||||
return [res[0], res[1]]
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// Example usage
|
||||
double latitude = 4832281.76
|
||||
double longitude = 12959065.3
|
||||
|
||||
// double[] mercatorPoint = bdLatlng2Mercator(latitude, longitude);
|
||||
// System.out.println("Mercator Point: " + Arrays.toString(mercatorPoint));
|
||||
|
||||
double[] latlngPoint = bdMercator2Latlng(longitude, latitude);
|
||||
System.out.println("Latlng Point: " + Arrays.toString(latlngPoint));
|
||||
}
|
||||
}
|
||||
+42
-12
File diff suppressed because one or more lines are too long
+20
-4
@@ -27,6 +27,8 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
|
||||
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
@@ -64,7 +66,7 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
for (def c : citys) {
|
||||
def allRets = []
|
||||
if(["上海","北京","广州","深圳","成都","天津","西安"].contains(c)){
|
||||
if(["上海","北京","广州","深圳","成都","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳"].contains(c)){
|
||||
continue;
|
||||
}
|
||||
index++
|
||||
@@ -263,9 +265,14 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||
if(!map.township){
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
def obj = findTownshipByBD(dist.id)
|
||||
if(obj){
|
||||
map.township = obj.township
|
||||
}else{
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
}
|
||||
}
|
||||
println map.township
|
||||
@@ -288,6 +295,15 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def findTownshipByBD(id){
|
||||
def q = new BasicDBObject()
|
||||
q.put("id", id)
|
||||
def obj = MAPPOI_CONN.findOne(q)
|
||||
return obj
|
||||
|
||||
}
|
||||
|
||||
|
||||
def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
|
||||
Reference in New Issue
Block a user