This commit is contained in:
2025-01-16 17:29:08 +08:00
parent 82ef32206f
commit e0aa49ebc3
23 changed files with 1237 additions and 151 deletions
@@ -3,6 +3,7 @@ package com.fangjia.core.util;
import java.util.HashMap;
import java.util.Map;
public class GeoUtils {
public static final String LAT ="lat";
@@ -258,6 +259,8 @@ public class GeoUtils {
return ret;
}
public static void main(String[] args){
Map<String, Double> location = gps84ToGcj02(31.218024629574998,121.36562884455);
System.out.println(location.toString());
+6
View File
@@ -537,6 +537,12 @@
<version>1.70</version> <!-- 确保使用最新版本 -->
</dependency>
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>1.18.2</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -4209,7 +4209,7 @@ class CommonDatautils {
static def findAllNewPoiDatas(city){
String sql = "select pname,cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
@@ -4222,6 +4222,8 @@ class CommonDatautils {
}
static def findOneThirdNetWorkOffice(city, region, name){
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
def map = [:]
@@ -0,0 +1,62 @@
package com.fangjia.script.gj
import groovy.json.JsonOutput
class ProxyHttpUtils {
static String proxyHost = 'tunnel2.qg.net';
static Integer proxyPort = 14186;
static String proxyUsername = '113BBC06';
static String proxyPassword = '3B2FBD05B0AF';
static String getProxyResult(targetUrl, data){
// 将 JSON 对象转换为字符串
String jsonData = new JsonOutput().toJson(data);
// 创建代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
// 创建 URL 对象
URL url = new URL(targetUrl);
// 创建连接对象
URLConnection connection = url.openConnection(proxy);
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST");
// 允许输出流
connection.setDoOutput(true);
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json");
// 发送请求参数
OutputStream outputStream = connection.getOutputStream();
outputStream.write(jsonData.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
// 读取响应
int responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
response.append(line);
}
reader.close();
return response.toString();
} else {
println "HTTP request failed with response code ${responseCode}";
}
return null;
}
}
@@ -0,0 +1,48 @@
package com.fangjia.script.gj.coresite
import java.net.HttpURLConnection
import java.net.URL
import java.io.OutputStream
// 目标服务器的URL
def targetUrlString = "http://data.zz.baidu.com/urls?site=sh.fangjia.com&token=UaKTemkcQ6I4YvSA"
// 需要POST的URL数据
def postData = "https://sh.fangjia.com/zoushi?cityName=%E4%B8%8A%E6%B5%B7&keyword=%E5%A4%A9%E5%B1%B1%E6%96%B0%E8%8B%91"
// 创建URL对象
URL url = new URL(targetUrlString)
// 打开连接
HttpURLConnection connection = (HttpURLConnection) url.openConnection()
connection.setDoOutput(true) // 表明我们将发送POST请求的输出数据
connection.setRequestMethod("POST") // 设置请求方法为POST
connection.setRequestProperty("User-Agent", "curl/7.12.1") // 设置User-Agent,尽管这在实际应用中可能并非必需
connection.setRequestProperty("Content-Type", "text/plain") // 设置内容类型,因为我们将发送纯文本数据
connection.setRequestProperty("Content-Length", Integer.toString(postData.getBytes().length)) // 设置内容长度
// 发送POST数据
//try (OutputStream os = connection.getOutputStream()) {
// byte[] input = postData.getBytes("utf-8")
// os.write(input, 0, input.length)
//}
outputStream = connection.getOutputStream();
outputStream.write(postData.getBytes());
outputStream.flush();
outputStream.close();
// 获取响应码
int responseCode = connection.getResponseCode()
println "POST Response Code :: $responseCode"
// 可选:读取响应内容
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"))
StringBuilder response = new StringBuilder()
String responseLine
while ((responseLine = reader.readLine()) != null) {
response.append(responseLine.trim())
}
println "Response Body :: ${response.toString()}"
// 关闭连接
connection.disconnect()
@@ -0,0 +1,202 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.json.JsonSlurper
import groovy.io.FileType
import org.bson.types.ObjectId
import java.util.regex.Pattern
class AddDevShapeData {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
def devAreaMap = [:]
static main(args) {
def task = new AddDevShapeData();
task.run()
// task.streetInfo()
}
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\111开发区\\'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
// dir.eachFileRecurse(FileType.FILES) { file ->
// process(file.absolutePath)
// }
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
} else {
println "The specified path is not a directory."
}
}
def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d->
// if(!d['I']){
def name = d['D']
def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A']))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
if(obj){
if(obj.operator){
d['H'] = obj.operator
}
if(obj.operate_result){
d['I'] = obj.operate_result
}
if(obj.m_date){
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
}
}
// }
devAreaMap.each {k,v->
if(name.contains(k)){
d['M'] = v.area_info
d['N'] = v.count
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
}
}
})
}
def process(file){
def retMap = getNewLocations(file)
if(retMap && retMap.gaode.size()>0){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+retMap.name+'.*$', Pattern.CASE_INSENSITIVE)))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
if(obj){
for(def i=0;i<retMap.gaode.size();i++){
def udata = new BasicDBObject()
def resultMap = [:]
def bd_resultMap = [:]
resultMap.type = "polygon"
resultMap.coordinates = [retMap.gaode[i]]
bd_resultMap.type = "polygon"
bd_resultMap.coordinates = [retMap.baidu[i]]
if(i>0){
obj.removeField("_id")
obj.put("create_time", new Date())
obj.put("block", retMap.name+"-地块"+i)
obj.put("shape", resultMap)
obj.put("m_date", new Date())
obj.put("bd_shape",bd_resultMap)
obj.put("operator", "外包")
obj.put( "operate_result" , "已处理")
DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{
udata.put("shape", resultMap)
udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "已处理")
udata.put("operator", "外包")
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
}
}
}
}
def getNewLocations(file){
def jsonObj = readJson(file)
def features = jsonObj.features
def gaode_locations = []
def baidu_locations = []
def name = null
def areaMap = [:]
if(features && features.size()==1){
name = features[0].properties.Name
def coordinates = features[0].geometry.coordinates
def index = 0
println name +" 有 "+coordinates.size()+" 块"
areaMap.count = coordinates.size()
def areas = []
if(name.contains("泉州经济技术开发区")){
for(def pointList : coordinates){
index ++
def gaode_coordinates = []
def bd_coordinates = []
def points = null
if(pointList.size()>1){
points = pointList
}else{
points = pointList[0]
}
double area = com.fangjia.script.util.GeoHelper.calculateArea(points)
println name +" 地块"+index + "围栏面积是: ${area} 平方公里"
areas << area
def locationstr = ""
for(def arrs : points){
def gaode_map = GeoUtils.gps84ToGcj02(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
gaode_coordinates << [gaode_map.get("lng"), gaode_map.get("lat")]
//此高德转百度 有偏差
def bd_map = GeoUtils.gcj02ToBd09(gaode_map.get("lat"), gaode_map.get("lng"))
// bd_coordinates << [bd_map.get("lng"), bd_map.get("lat")]
// 高德转百度 通过接口有次数限制
// def baidu = BaiDuApiUtils.geoconv(gaode_map.get("lng"), gaode_map.get("lat"), 3)
// bd_coordinates << [baidu.x, baidu.y]
locationstr = arrs[0].toString()+","+arrs[1].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()), 6)
if(bdpoint){
bd_coordinates << bdpoint
}
}
//整个一串一起传超过4029会报错
// bd_coordinates = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()-1), 6)
gaode_locations << gaode_coordinates
if(bd_coordinates){
baidu_locations << bd_coordinates
}else{
println name +" 转换有问题"
}
}
areaMap.area_info = areas.join("\n")
}
}else{
println file +" 有多个"
}
devAreaMap.put(name, areaMap)
def retMap = [:]
retMap.name = name
retMap.gaode = gaode_locations
retMap.baidu = baidu_locations
return retMap
}
def readJson(def file){
def jsonSlurper = new JsonSlurper()
def jsonFile = new File(file)
def jsonText = jsonFile.text
def jsonObj = jsonSlurper.parseText(jsonText)
return jsonObj
}
def streetInfo(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx", {d->
def count = dev_mapping_CONN.count(new BasicDBObject("name", d['E']).append("type", "乡镇街道"))
d['L'] = count
})
}
}
@@ -23,19 +23,19 @@ import java.util.concurrent.atomic.AtomicLong
//补充库中没有详细地址的小区
class AddDistrictAddress extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
// def fullCityMap = CommonDatautils.queryFullNameCity()
// def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
//
// DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
// public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
//
//
// public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
//
// def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
// DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
//
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new AddDistrictAddress(), args)
@@ -43,7 +43,16 @@ class AddDistrictAddress extends BaseBatch{
@Override
public void process(Object args) {
// def citys = CommonDatautils.findAllCity()
// addNoAddressDist() //为不精确的地址通过地图补充地址
// mapOtherAddress() //有精确地址的寻找其它地址
// addressSelect()
addrUpdateToDB()
}
def addNoAddressDist(){
// def citys = CommonDatautils.findAllCity()
// def index = 0
// def allrets = []
// for(def c : citys){
@@ -65,9 +74,95 @@ class AddDistrictAddress extends BaseBatch{
// allrets.clear()
// }
// }
updateDistAddr()
}
updateDistAddr()
def mapOtherAddress(){
def citys = CommonDatautils.findAllCity()
def index = 0
def allrets = []
for(def c : citys){
index ++
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address", "addr_alias"])
def radInfoDists = haveRoadInfoDistricts(districts)
println c +" "+index +" 原始小区量:"+districts.size() +" 有详细地址量:"+radInfoDists.size()
radInfoDists = baiduMatch(c, radInfoDists)
radInfoDists = gaodeMatch(c, radInfoDists)
def rets = []
rets << ["id", "城市", "区域", "名称", "地址","地址别名", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
for(def d : radInfoDists){
def addr_alias_str = '';
if(d.addr_alias){
addr_alias_str = d.addr_alias.join(" ")
}
rets << [d._id.toString(), d.city, d.region, d.name, d.address, addr_alias_str,d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
}
allrets.addAll(rets)
if(allrets.size()>50000){
ExcelUtils.write("D:\\数据\\小区\\小区精确地址补充"+index+".xlsx", allrets, "补充地址")
allrets.clear()
}
}
}
def addressSelect(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-all-2.xlsx",{d->
if(d['G']){
d['O'] = AddressUtil.formatAddress(d['B'], d['G'])
}
if(d['I']){
d['P'] = AddressUtil.formatAddress(d['B'], d['I'])
}
})
}
def addrUpdateToDB(){
def updateDist = 0
def addAddrCount = 0
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-入库-2.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def oldAddrAliasCount = 0
if(dist.addr_alias){
oldAddrAliasCount = dist.addr_alias.size()
}
def new_addr_alias = null
if(d['K'].contains("新增百度地址") || d['K'].contains("双地图新增")){
new_addr_alias = getNewAddressAlias(d['L'], dist.address, dist.addr_alias)
}
if(d['K'].contains("新增高德地址") || d['K'].contains("双地图新增")){
new_addr_alias = getNewAddressAlias(d['M'], dist.address, dist.addr_alias)
}
if(new_addr_alias){
updateDist++
println updateDist
addAddrCount = addAddrCount + new_addr_alias.size()-oldAddrAliasCount
def udata = new BasicDBObject()
udata.put("addr_alias", new_addr_alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
}
}
})
println "修改小区数:"+updateDist
println "新增地址数:"+addAddrCount
}
def getNewAddressAlias(new_addr, old_addr, dist_addr_alias){
def arrs = new_addr.split(";")
def addr_alias = []
if(dist_addr_alias){
addr_alias = dist_addr_alias
}
for(def address : arrs){
if(old_addr!=address && !addr_alias.contains(address)){
addr_alias << address
}
}
return addr_alias
}
def updateDistAddr(){
@@ -108,6 +203,19 @@ class AddDistrictAddress extends BaseBatch{
return rets;
}
def haveRoadInfoDistricts(districts){
def rets = []
for(def d : districts){
if(d.address){
def flag = DataHandleUtils.isStandardRoad(d.address)
if(flag){
rets << d
}
}
}
return rets;
}
def baiduMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
@@ -166,7 +274,8 @@ class AddDistrictAddress extends BaseBatch{
retMap.old_addr = temp_address
}
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
retMap.new_addr = nineAddrSplit(fullCityName, temp_address)
continue
}
}
@@ -175,6 +284,21 @@ class AddDistrictAddress extends BaseBatch{
}
def nineAddrSplit(city, addr){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
def address = ""
if (jsonObject) {
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
address = jsonObject.get("street")
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
address = address + jsonObject.get("streetNumber")
}
}
return address
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
@@ -250,7 +374,8 @@ class AddDistrictAddress extends BaseBatch{
if(temp_addr){
retMap.old_addr = temp_addr
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
retMap.new_addr = nineAddrSplit(fullCityName, temp_addr)
continue
}
}
@@ -57,7 +57,7 @@ class AddDistrictShapeBatch extends BaseBatch{
//
// }
checkRangeCount()
// checkRangeCount()
// synch100()
}
@@ -84,6 +84,7 @@ class DevProcess {
// task.addQiDevResult() //获取齐梁数据结果
// task.updateDevShape()//修改开发区结构
}
@@ -439,30 +440,33 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
if(!d['B'] && d['A']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
if(!d['E'] && d['D']){
def q = new BasicDBObject()
// q.put("name", d['A'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['A']]))
def name = d['A'].replace('经济技术开发区','经济开发区')
// q.put("name", d['D'])
q.put("city",d['C'])
// q.put('alias', new BasicDBObject('$in',[d['D']]))
def name = d['D'] //.replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['I'] = "库中有开发区-相似2"
d['G'] = "相似"
// d['K'] = obj.name
// d['C'] = obj._id.toString()
d['B'] = obj.name
// if(obj.name == d['A']){
// d['E'] = "相同"
d['E'] = obj.name
// if(obj.name == d['D']){
// d['G'] = "相同"
// }else{
// d['E'] = "包含"
// d['G'] = "包含"
// }
if(obj.shape){
d['F'] = "是"
}
// d['E'] = "别名"
// d['F'] = obj.name
}
}
},1);
},0);
}
def calAddrDistince(){
@@ -0,0 +1,113 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.ExcelUtil
import com.fangjia.script.gj.ProxyHttpUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
class DevQlCheck {
//http://146.56.217.44:8088/uams/index
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static main(args) {
def task = new DevQlCheck()
task.run()
}
def run(){
def devName = "福建龙海经济开发区"
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",devName))
def pois = readCSV("C:\\Users\\gaojuan\\Downloads\\漳州市POI.csv")
def devPois = []
devPois << ["省", "城市", "区域", "名称", "地址", "类型", "经度", "纬度", "齐梁开发区"]
for(def poi : pois){
def flag = InsidePolygon.isInArea(poi.lng.toString().toDouble(), poi.lat.toString().toDouble(), InsidePolygon.createPoint4(obj.shape.coordinates))
if(flag){
devPois << [poi.province, poi.city, poi.district, poi.name, poi.address, poi.cat, poi.lng, poi.lat]
}
}
println "共:"+devPois.size()+" 个"
ExcelUtils.write("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx", devPois, "齐梁开发区")
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx",{d->
def address = d['B']+d['C']+d['D']
// if(d['E']){
// address = address + d['E']
// }
Thread.sleep(3000)
def s = getQlResult(address)
d['I'] = s
index ++
println index+" "+ s
})
}
def getQlResult(address){
def params = [
'address': address,
'type': '1' //1:对公 0:对私
]
try{
def str = ProxyHttpUtils.getProxyResult(targetUrl, params);
println address +" "+ str
JSONObject result = JSONObject.fromObject(str)
if(result && result.code==20000){
def data = result.data
if(data){
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
return s
}
}
}
}catch(Exception e){
println address +" "+e.getMessage()
}
return null
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
try{
o.put("province", arrs[0])
o.put("city", arrs[1])
o.put("district", arrs[2])
o.put("name", arrs[3])
o.put("township", arrs[4])
o.put("address", arrs[5])
o.put("cat", arrs[6])
o.put("lng", arrs[7])
o.put("lat", arrs[8])
o.put("data_source", arrs[9])
rets << o
}catch(Exception e){
println filename+" 文件异常"
}
}
}
return rets;
}
}
@@ -32,7 +32,7 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
//
@@ -167,7 +167,7 @@ class DevUpdateTestBatch {
println d['A'] +" 不存在"
}
}
},0)
},1)
}
@@ -1,12 +1,14 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -28,14 +30,24 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
task.addOldMatch2()
// task.addOldMatch2()
// task.runDb()
// task.hainanMatch()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
// task.runMatch()
// task.checkDist()
task.addPrice()
}
def addPrice(){
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
}
def addOldMatch(){
@@ -94,6 +106,46 @@ class GXDdistrictMatch {
}
def hainanMatch(){
def fileName = "D:\\数据\\小区\\海南未匹配地址-1.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['D'] && d['F'] ){//
// def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['I'])
// if(jsonObject){
//// if(jsonObject.get("district") && jsonObject.get("district")!='null'){
//// d['AO'] = jsonObject.get("district")
//// }
// if(jsonObject.get("province") && jsonObject.get("province")!="null"){
// d['AP'] = jsonObject.get("province")
// }
// if(jsonObject.get("city") && jsonObject.get("city")!="null"){
// d['AI'] = jsonObject.get("city")
// }
// }
def formatCity = RegionData.formatCity(d['F'])
def fullCityObj = fullCityMap.get(formatCity)
if(fullCityObj){
// def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
def obj = findDatasByCityAndName(formatCity, d['D'].replaceAll('[·\\(\\)()]|·', ''))
if(obj){
d['U'] = "楼盘抓取库"
d['V'] = obj.city
d['W'] = obj.region
d['X'] = obj.name
d['Y'] = obj.property_type
d['Z'] = obj.address
def loc = obj.location
if(loc){
d['AA'] = loc.x
d['AB'] = loc.y
}
}
}
}
})
}
def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
@@ -238,43 +290,41 @@ class GXDdistrictMatch {
def saveToDB(){
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
def fileName = "D:\\数据\\小区\\海南新增小区.xlsx"
ExcelUtils.modify(fileName, { d ->
if(d['A']){
if(d['A'] && d['C']){
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def alias = []
if(d['G'] && name!=d['G']){
alias << d['G']
}
// def alias = []
// if(d['G'] && name!=d['G']){
// alias << d['G']
// }
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
// alias << d['AI']
// }
if(alias){
udata.put("alias", alias)
}
// if(alias){
// udata.put("alias", alias)
// }
def addr = AddressUtil.formatAddress(d['A'], d['D'])
def addr = AddressUtil.formatAddress(d['A'], d['E'])
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("status", 0)
udata.put("name_cat", 0)
udata.put("cat", "district")
udata.put("username", "高娟")
// udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
// if(d['AH']){
// type = d['AH']
// }else{
// type = d['Y']
// }
if(d['D']){
type = d['D']
}
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
@@ -282,26 +332,50 @@ class GXDdistrictMatch {
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
udata.put("estate_type2", estateType2)
if(d['F']){
Long lng = d['F']*10000000000
udata.put("lng", lng)
}
if(d['E']){
Long lat = d['E'] * 10000000000
Long lat = d['G'] * 10000000000
udata.put("lat", lat)
}
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
def opt = "add"
if(s && s=='成功'){
saveOptLog(udata.city, udata.region, udata.name, opt)
udata.put("lng", lng)
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(d['F'].toString().toDouble(), d['G'].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
if(lng2){
Long lng2_new = lng2 *10000000000L
Long lat2_new = lat2 *10000000000L
udata.put("lng2", lng2_new)
udata.put("lat2", lat2_new)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map",markMap)
// udata.put("is_new", 0)
DbUtils.seaweed.save(udata)
saveOptLog(udata.city, udata.region, udata.name, "add")
// def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println name
// def opt = "add"
// if(s && s=='成功'){
// saveOptLog(udata.city, udata.region, udata.name, opt)
// }
}
})
}
def checkDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\海南未匹配地址-1.xlsx", { d ->
if(d['V'] && d['W'] && d['X']){
matching(d['V'], d['W'], d, d['X'], null, null)
}
})
}
def match(city, region, d){
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
@@ -391,7 +465,7 @@ class GXDdistrictMatch {
if(rets && rets.size()>=1){
d = setRowValue(rets[0], d, "名称在地址中"+str)
}else{
if (d['D']) {
if (d['D'] && alias) {
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
if (rets) {
if (rets.size() >= 1) {
@@ -449,14 +523,14 @@ class GXDdistrictMatch {
def setRowValue(dbo, d, remark){
if (dbo) {
d['V'] = dbo.city
d['W'] = dbo.region
d['X'] = dbo.name
d['Y'] = dbo.estate_type2
d['Z'] = dbo.status
d['AA'] = dbo.name_cat
d['U'] = remark
d['AH'] = dbo.address
// d['V'] = dbo.city
// d['W'] = dbo.region
// d['X'] = dbo.name
// d['Y'] = dbo.estate_type2
d['AK'] = dbo.status
// d['AA'] = dbo.name_cat
d['AJ'] = remark
// d['AH'] = dbo.address
}
return d;
}
@@ -510,6 +584,33 @@ class GXDdistrictMatch {
return null;
}
def findDatasByCityAndName(def city, def name){
def sql = "select * from community where city='${city}' and name='${name}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
println d.property_type
})
if(rets){
def obj = null
for(def r : rets){
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
obj = r
break
}
}
if(obj){
return obj;
}else{
return rets[0];
}
}
return null;
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
@@ -467,7 +467,7 @@ class NewGaoDePoiBatch extends BaseBatch{
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def map = new HashMap()
ElasticUtils89.executeQuery("192.168.0.89", "map_poi_2024", queryBuilder, { String id, Map d->
def district = ""
def district = "嘉峪关市"
if(d.district){
district = d.district
}
@@ -477,11 +477,12 @@ class NewGaoDePoiBatch extends BaseBatch{
}
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("district", district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
regionMap.put(d.name.replace("-","").replace("¡¤",""), o)
// map.put(district, regionMap)
map.put(district, regionMap)
})
return map
@@ -6,7 +6,7 @@ import com.fangjia.script.util.ExcelUtils
class PoiNineSplitBatch{
static main(args) {
public static void main(args) {
def task = new PoiNineSplitBatch()
task.run()
}
@@ -3,7 +3,9 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
@@ -12,6 +14,13 @@ import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -22,6 +31,7 @@ class TownStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def updateList = []
def addAliasList = []
@@ -38,7 +48,8 @@ class TownStandardBatch extends BaseBatch{
def TownStandard(){
// townStandardByCat("office")
townStandardByPoi()
// townStandardByPoi()
addTownByTownShape()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
@@ -69,11 +80,15 @@ class TownStandardBatch extends BaseBatch{
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
//浙江,江苏,安徽
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
def runProvinces = ["福建省","浙江省","江苏省","安徽省","山东省","甘肃省"]
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index++
def data = fullCityMap.get(city)
// if(runProvinces.contains(data.province)){
// continue
// }
def fullCityName = null
if(data){
fullCityName = data.city
@@ -95,11 +110,16 @@ class TownStandardBatch extends BaseBatch{
def rets = readPoiCsv(fileName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data)
}else{
// haveStreetProcess(city, set, cat)
}
}
if(updateList.size()>50000){
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-${index}.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名-${index}.xlsx", addAliasList, "数据")
updateList.clear()
}
}else{
println fileName+" 文件没找到"
}
@@ -107,8 +127,7 @@ class TownStandardBatch extends BaseBatch{
println "没找到相应的全称:"+city
}
}
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-最后.xlsx", updateList, "数据")
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
@@ -168,7 +187,7 @@ class TownStandardBatch extends BaseBatch{
}
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data){
def results = []
if(fullCityName){
for(def d : rets){
@@ -221,21 +240,21 @@ class TownStandardBatch extends BaseBatch{
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.province, d.city, d.district, d.name,d.township, d.status, updateTown]
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
// results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
results << m
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.province, d.city, d.district, d.name, d.township, d.status, ""]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
}
}
@@ -244,10 +263,10 @@ class TownStandardBatch extends BaseBatch{
}
}
// if(results.size()>0){
// updateByEs(results)
// results.clear()
// }
if(results.size()>0){
updateByEs(results)
results.clear()
}
println fullCityName +" 处理完成"
}
@@ -322,18 +341,194 @@ class TownStandardBatch extends BaseBatch{
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
try{
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}catch(Exception e){
println arrs
}
}
}
return rets;
}
def readCSV2(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
def o = new BasicDBObject()
try{
o.put("id", arrs[1])
o.put("province", arrs[2])
o.put("city", arrs[3])
o.put("district", arrs[4])
o.put("lng", arrs[5])
o.put("lat", arrs[6])
o.put("data_source", arrs[7])
rets << o
}catch(Exception e){
println arrs
}
}
index++
}
return rets;
}
def addTownByTownShape(){
def i = 0
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式"]
while(i<50){
def fileName = "/home/goojia/gj/new_poi/input/no_township_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
def rets = readCSV2(fileName)
def rows = addTownshipBatch(rets)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
println fileName +" 运行完成"
}else{
println fileName +" 不存在"
}
i++
}
}
def addTownshipBatch(def rets){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
if (rMap) {
def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
def set = rMap.keySet()
println totalCount.get() +" "+r.city + " " + set.size()
if (set.size() >= 1) {
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source]
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return results
}
def addTown(saveObj, townMap, location) {
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
def towns = townMap.get(saveObj.district)
if (towns) {
for (def town : towns) {
if (town.shape && town.shape.coordinates) {
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if (flag) {
saveObj.township = town.street
saveObj.town_source = "同区域"
break
}
}
}
}
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
if(!saveObj.township){
def townNames = townMap.keySet()
for(def townName : townNames){
towns = townMap.get(townName)
for (def town : towns) {
if (town.shape && town.shape.coordinates) {
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if (flag) {
saveObj.township = town.street
saveObj.new_district = town.mzb_district_name
saveObj.town_source = "忽略区域"
break
}
}
}
if(saveObj.township){
break;
}
}
}
//通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
// def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
// if(gaodeMap && gaodeMap.township){
// saveObj.township = gaodeMap.township
// saveObj.new_district = gaodeMap.district
// saveObj.town_source = "高德API"
// }
// }
return saveObj
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def stop = false
def count = 0
while(!stop){
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
city = addressComponent.city
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
}
@@ -38,16 +38,16 @@ class TownshipStrandUpdate {
static main(args) {
def task = new TownshipStrandUpdate()
// task.process()
// task.updateTown()
task.process()
task.updateTown()
// task.stat()
task.updateDiffRegionTown()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// println new Date().getTime()
// task.updateSpecCityTown()
}
@@ -67,7 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
]
static main(args) {
runBatch(new DistrictFieldAddBatch(), args)
// runBatch(new DistrictFieldAddBatch(), args)
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
// if(dist){
// d['L'] = dist.loudong_count
// d['M'] = dist.household_count
// d['Q'] = dist.estate_type2
// }
def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
if(!list && dist){
def alias = dist.alias
for(def name : alias){
list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
if(list){
d['Z'] = "别名"
}
}
}else{
d['Z'] = "名称"
}
if(list){
def obj = list[0]
if(obj.loudong_cnt){
d['W'] = obj.loudong_cnt
}
if(obj.households){
d['X'] = obj.households
}
d['Y'] = obj.url
}
})
}
@Override
@@ -361,6 +393,15 @@ class DistrictFieldAddBatch extends BaseBatch{
return list
}
static def findEsDatasByCityAndName(c, district, name, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' and region='${district}' and name='${name}'", {r->
list << r
})
return list
}
def fieldUpdate(total,updateRets){
def countMap = [:]
@@ -398,4 +398,32 @@ class FangjiaClientUtils {
return jsonObj;
}
/**
* 经纬度转换
* 1、高德转百度 2、高德转WGS84 3、百度转高德 4、WGS84转高德 5、百度转WGS84 6、WGS84转百度
*/
public static def convert(String locationstr, type){
def jsonObj = null
def content = null
def points = []
try {
def url = "http://192.168.0.19:31080/FangJiaNlpService/convert?locations="+URLEncoder.encode(locationstr,"UTF-8")+"&coordsys=${type}"
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
if(jsonObj && jsonObj.locations){
def locations = jsonObj.locations
for(def s : locations){
def arrs = s.split(",")
points = [arrs[0].toString().toDouble(), arrs[1].toString().toDouble()]
}
}
return points
} catch (Exception e) {
println e.getMessage()
}
return null;
}
}
@@ -13,18 +13,18 @@ class MsbPocTest {
task.run();
}
def run(){
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
def inputfile = "D:\\数据\\建行\\房产估值\\房讯通10月价格核查_1126-FXT提交结果20241205.xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
// matchOld(inputfile)
matchOld(inputfile)
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
// test("D:\\数据\\美术宝\\test2.xlsx")
// cityPrice()
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
addCCBPrice(inputfile)
// addCCBPrice(inputfile)
}
def cityPrice(){
@@ -69,24 +69,24 @@ class MsbPocTest {
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Y'] && row['G']){
if(!row['AD'] ){
def city = row['C']
def region = null //row['D']
def name = row['E']
def address = row['G']
def name = null// row['E']
def address = row['D']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("propertyType")
row['AD'] = jsonObject.get("id")
row['AE'] = jsonObject.get("city")
row['AF'] = jsonObject.get("district")
row['AG'] = jsonObject.get("name")
row['AH'] = jsonObject.get("propertyType")
// row['X'] = jsonObject.get("credit")
}
}
})
},1)
}
@@ -114,7 +114,7 @@ class MsbPocTest {
if(!row['Y']){
def city = row['C']
def region = null //row['B']
def address = row['G'] //row['G']//row['F']
def address = row['D'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
@@ -125,7 +125,7 @@ class MsbPocTest {
row['AB'] = jsonObject.get("name")
}
}
})
},1)
}
@@ -38,9 +38,9 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
def filePath = "D:\\Êý¾Ý\\¼Û¸ñ\\»ÝÖÝ.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
@@ -174,26 +174,28 @@ class AddPriceTest {
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['U']){
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(d['A']){
def cityId = cityCodeDataMap.get(d['B'])
def map = findAvgPrice(d['A'], cityId)
if(map){
d['H'] = map.avg_price
d['I'] = map.calc_type
d['I'] = map.avg_price
d['J'] = map.month
// d['H'] = map.calc_type
// d['AM'] = map.credit_value
}
// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId)
// if(newPriceMap){
// d['Q'] = newPriceMap.avg_price
// }
def newPriceMap = findNewFjwAvgPrice(d['A'], cityId)
if(newPriceMap){
d['G'] = newPriceMap.avg_price
d['H'] = newPriceMap.month
}
}
})
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -213,7 +215,7 @@ class AddPriceTest {
def findNewFjwAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance235.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -270,14 +272,14 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['B']) {
if (!d['H']) {
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
if(gxdPrice){
d['B'] = gxdPrice.price
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
d['H'] = gxdPrice.price
d['I'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
}
}
@@ -305,11 +307,11 @@ class AddPriceTest {
ExcelUtils.multiThreadingModify(path,{d->
// if(!d['L']){
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['AJ'])
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
d['E'] = dist.name
// d['E'] = dist.name
// d['D'] = dist.status
// d['E'] = dist.name_cat
// d['F'] = dist.estate_type2
@@ -318,9 +320,9 @@ class AddPriceTest {
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['G'] = dbo.avg_price
d['E'] = dbo.avg_price_2b
// d['X'] = dbo.cal_type
// d['E'] = dbo.avg_price_2b
// d['E'] = dbo.avg_price
}
}
// }
@@ -1,4 +1,8 @@
package com.fangjia.script.util
import java.text.DecimalFormat
class GeoHelper {
@@ -60,10 +64,159 @@ class GeoHelper {
// return GetLatLon(a, distance, angle);
// }
//
public static void main(String[] args){
GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
}
public static haversine(lat1, lon1, lat2, lon2) {
def R = 6371; // 地球半径(公里)
def lat1Rad = lat1 * (Math.PI / 180);
def lat2Rad = lat2 * (Math.PI / 180);
def deltaLat = (lat2 - lat1) * (Math.PI / 180);
def deltaLon = (lon2 - lon1) * (Math.PI / 180);
def a = Math.sin(deltaLat / 2) ** 2 +
Math.cos(lat1Rad) * Math.cos(lat2Rad) *
Math.sin(deltaLon / 2) ** 2;
def c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
return R * c; // 返回距离(公里)
}
public static calculateArea(coords) {
def area = 0;
def n = coords.size();
for (def i = 0; i < n; i++) {
def j = (i + 1) % n;
area += coords[i][0] * coords[j][1];
area -= coords[j][0] * coords[i][1];
}
area = Math.abs(area) / 2; // 平方度
// 转换为平方公里
def conversionFactor = 111.32; // 大约每度经纬度的公里数(考虑到地球的曲率)
def df = new DecimalFormat("#.000000")
return Double.parseDouble(df.format(area * conversionFactor * conversionFactor)) // 返回面积(平方公里)
// return area * conversionFactor * conversionFactor
}
public static void main(String[] args){
// GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
// GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
List<List<Double>> coordinates = [
[
117.61878540582734,
24.667331307258223
],
[
117.61400840589488,
24.670008244935104
],
[
117.60936140565889,
24.671837202755682
],
[
117.60834440585768,
24.668667276729821
],
[
117.60696440583342,
24.667256309201207
],
[
117.60517040553236,
24.667762297657362
],
[
117.60272340547806,
24.660100475272998
],
[
117.60254340555321,
24.658630509428463
],
[
117.60206940590035,
24.654781598282568
],
[
117.60254840557604,
24.652731645907636
],
[
117.60346440599301,
24.651555673447035
],
[
117.6070314057805,
24.652578649754957
],
[
117.61605140571805,
24.651939664372833
],
[
117.6156454058372,
24.654165613124405
],
[
117.61559340595863,
24.655023593107103
],
[
117.61535440558418,
24.658990501065318
],
[
117.61662340528017,
24.659352492642654
],
[
117.61678740531192,
24.661365445721003
],
[
117.61472940542001,
24.661623439403797
],
[
117.61462940586114,
24.661837434759093
],
[
117.61470140583106,
24.662665415879701
],
[
117.61487240553515,
24.663165403873506
],
[
117.61522940591078,
24.663494396452791
],
[
117.6157014059138,
24.663523395394918
],
[
117.61627240565099,
24.663438397829282
],
[
117.61693540544979,
24.663181403867725
],
[
117.61878540582734,
24.667331307258223
]
]
double area = calculateArea(coordinates)
println "The area of the polygon is: ${String.format("%.2f", area)} square kilometers"
}
}
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //±¦½£
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //±¦½£:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995£ºdfc2245bac8a7b2d5b62e060e59d4eca
/**