Merge branch 'script_gj' into 'master'

Suggestion乡镇街道从map_poi中查,提升速度

See merge request !789
This commit is contained in:
2024-01-15 15:40:47 +08:00
5 changed files with 229 additions and 52 deletions
@@ -28,8 +28,8 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()//删除第三方
task.delDealByPe()
task.deleteThirdPartyData()//删除第三方
task.delDealByPe()//删除成交
// task.delDealData()
// task.exportThirdParty()
@@ -6,6 +6,7 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.CNNMFilter
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.BaiduCoordinateConverter
import com.fangjia.script.util.CosineSimilarAlgorithm
import net.sf.json.JSONObject
import net.sf.json.groovy.JsonSlurper
@@ -389,49 +390,32 @@ class OpenApiUtils {
for (def loc : json.content) {
String name = formatName(address)
if(loc.uid) {
def bean = [:]
JSONObject uidInfo = BaiDuApiUtils.getDetailByUid(loc.uid)
if(uidInfo != null) {
def baiduRegion = uidInfo.get("result").area
def baiduCity = uidInfo.get("result").city
rets = createData(rets, uidInfo, loc, region, name, address)
}else{
def baiduRegion = loc.area_name.replace(loc.city_name,"")
def baiduCity = loc.city_name
baiduCity = RegionData.formatCity(baiduCity)
def newName = formatName(loc.name)
if(baiduRegion.contains(region) && (name== newName || address==newName)){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
Double lng2 = lo.get("lng")
Double lat2 = lo.get("lat")
bean.lng2 = lng2
bean.lat2 = lat2
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(score1 || score2){
def bean = [:]
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
def lng = gmap.get("lng")
def lat = gmap.get("lat")
bean.lng = lng
bean.lat = lat
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
def locArrs = BaiduCoordinateConverter.bdMercator2Latlng(loc.diPointX/100.0, loc.diPointY/100.0)
if(locArrs){
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0], locArrs[1])
bean.lng = gmap.get("lng")
bean.lat = gmap.get("lat")
}
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
}else{
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
rets << bean;
}
bean.estateType = loc.std_tag
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+loc.name+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(score1 || score2){
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
}
}
@@ -444,6 +428,67 @@ class OpenApiUtils {
return rets;
}
def static createData(def rets, def uidInfo, def loc, region, name, address){
def bean = [:]
def baiduRegion = uidInfo.get("result").area
def baiduCity = uidInfo.get("result").city
baiduCity = RegionData.formatCity(baiduCity)
def newName = formatName(loc.name)
if(baiduRegion.contains(region) && (name== newName || address==newName)){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
Double lng2 = lo.get("lng")
Double lat2 = lo.get("lat")
bean.lng2 = lng2
bean.lat2 = lat2
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
def lng = gmap.get("lng")
def lat = gmap.get("lat")
bean.lng = lng
bean.lat = lat
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
}
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
}else{
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
rets << bean;
}
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(score1 || score2){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
bean.lng2 = lo.get("lng")
bean.lat2 = lo.get("lat")
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
def gmap = GeocodeUtils.geoconvBaiduToGoogle(bean.lng2, bean.lat2)
bean.lng = gmap.get("lng")
bean.lat = gmap.get("lat")
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
}
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
}
return rets;
}
def static formatName(String name) {
if(name.indexOf("·") != -1) {
name = name.replaceAll("·", "")
@@ -0,0 +1,86 @@
package com.fangjia.script.gj.village
import java.util.Arrays;
class BaiduCoordinateConverter {
static double[] array1 = [75, 60, 45, 30, 15, 0]
static double[][] array2 = [
[-0.0015702102444, 111320.7020616939, 1704480524535203.0, -10338987376042340.0, 26112667856603880.0, -35149669176653700.0, 26595700718403920.0, -10725012454188240.0, 1800819912950474.0, 82.5],
[0.0008277824516172526, 111320.7020463578, 647795574.6671607, -4082003173.641316, 10774905663.51142, -15171875531.51559, 12053065338.62167, -5124939663.577472, 913311935.9512032, 67.5],
[0.00337398766765, 111320.7020202162, 4481351.045890365, -23393751.19931662, 79682215.47186455, -115964993.2797253, 97236711.15602145, -43661946.33752821, 8477230.501135234, 52.5],
[0.00220636496208, 111320.7020209128, 51751.86112841131, 3796837.749470245, 992013.7397791013, -1221952.21711287, 1340652.697009075, -620943.6990984312, 144416.9293806241, 37.5],
[-0.0003441963504368392, 111320.7020576856, 278.2353980772752, 2485758.690035394, 6070.750963243378, 54821.18345352118, 9540.606633304236, -2710.55326746645, 1405.483844121726, 22.5],
[-0.0003218135878613132, 111320.7020701615, 0.00369383431289, 823725.6402795718, 0.46104986909093, 2351.343141331292, 1.58060784298199, 8.77738589078284, 0.37238884252424, 7.45]
]
static double[] array3 = [12890594.86, 8362377.87, 5591021, 3481989.83, 1678043.12, 0];
static double[][] array4 = [
[1.410526172116255e-8, 0.00000898305509648872, -1.9939833816331, 200.9824383106796, -187.2403703815547, 91.6087516669843, -23.38765649603339, 2.57121317296198, -0.03801003308653, 17337981.2],
[-7.435856389565537e-9, 0.000008983055097726239, -0.78625201886289, 96.32687599759846, -1.85204757529826, -59.36935905485877, 47.40033549296737, -16.50741931063887, 2.28786674699375, 10260144.86],
[-3.030883460898826e-8, 0.00000898305509983578, 0.30071316287616, 59.74293618442277, 7.357984074871, -25.38371002664745, 13.45380521110908, -3.29883767235584, 0.32710905363475, 6856817.37],
[-1.981981304930552e-8, 0.000008983055099779535, 0.03278182852591, 40.31678527705744, 0.65659298677277, -4.44255534477492, 0.85341911805263, 0.12923347998204, -0.04625736007561, 4482777.06],
[3.09191371068437e-9, 0.000008983055096812155, 0.00006995724062, 23.10934304144901, -0.00023663490511, -0.6321817810242, -0.00663494467273, 0.03430082397953, -0.00466043876332, 2555164.4],
[2.890871144776878e-9, 0.000008983055095805407, -3.068298e-8, 7.47137025468032, -0.00000353937994, -0.02145144861037, -0.00001234426596, 0.00010322952773, -0.00000323890364, 826088.5]
];
public static double[] convertor(double x, double y, double[] param) {
double t = param[0] + param[1] * Math.abs(x);
double cc = Math.abs(y) / param[9];
double cf = param[2] + param[3] * cc + param[4] * cc * cc + param[5] * cc * cc * cc + param[6] * cc * cc * cc * cc + param[7] * cc * cc * cc * cc * cc + param[8] * cc * cc * cc * cc * cc * cc;
t *= (x < 0) ? -1 : 1;
cf *= (y < 0) ? -1 : 1;
return [t, cf];
}
public static double[] bdLatlng2Mercator(double lat, double lng) {
double[] arr = null;
double nLat = (lat > 74) ? 74 : lat;
nLat = (nLat < -74) ? -74 : nLat;
for (int i = 0; i < array1.length; i++) {
if (lat >= array1[i]) {
arr = array2[i];
break;
}
}
if (arr == null) {
for (int i = array1.length - 1; i >= 0; i--) {
if (lat <= -array1[i]) {
arr = array2[i];
break;
}
}
}
if (arr == null) {
throw new IllegalArgumentException("Invalid latitude");
}
double[] res = convertor(lng, lat, arr);
return [res[0], res[1]]
}
public static double[] bdMercator2Latlng(double x, double y) {
double[] arr = null;
double npX = Math.abs(x);
double npY = Math.abs(y);
for (int i = 0; i < array3.length; i++) {
if (npY >= array3[i]) {
arr = array4[i];
break;
}
}
if (arr == null) {
throw new IllegalArgumentException("Invalid y value");
}
double[] res = convertor(npX, npY, arr);
return [res[0], res[1]]
}
public static void main(String[] args) {
// Example usage
double latitude = 4832281.76
double longitude = 12959065.3
// double[] mercatorPoint = bdLatlng2Mercator(latitude, longitude);
// System.out.println("Mercator Point: " + Arrays.toString(mercatorPoint));
double[] latlngPoint = bdMercator2Latlng(longitude, latitude);
System.out.println("Latlng Point: " + Arrays.toString(latlngPoint));
}
}
File diff suppressed because one or more lines are too long
@@ -27,6 +27,8 @@ class SuggestionDataSynch extends BaseBatch{
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
@@ -64,7 +66,7 @@ class SuggestionDataSynch extends BaseBatch{
for (def c : citys) {
def allRets = []
if(["上海","北京","广州","深圳","成都","天津","西安"].contains(c)){
if(["上海","北京","广州","深圳","成都","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳"].contains(c)){
continue;
}
index++
@@ -263,9 +265,14 @@ class SuggestionDataSynch extends BaseBatch{
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
def obj = findTownshipByBD(dist.id)
if(obj){
map.township = obj.township
}else{
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
}
}
println map.township
@@ -288,6 +295,15 @@ class SuggestionDataSynch extends BaseBatch{
}
def findTownshipByBD(id){
def q = new BasicDBObject()
q.put("id", id)
def obj = MAPPOI_CONN.findOne(q)
return obj
}
def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)