update
This commit is contained in:
@@ -3,6 +3,7 @@ package com.fangjia.core.util;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
public class GeoUtils {
|
||||
|
||||
public static final String LAT ="lat";
|
||||
@@ -258,6 +259,8 @@ public class GeoUtils {
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static void main(String[] args){
|
||||
Map<String, Double> location = gps84ToGcj02(31.218024629574998,121.36562884455);
|
||||
System.out.println(location.toString());
|
||||
|
||||
@@ -537,6 +537,12 @@
|
||||
<version>1.70</version> <!-- 确保使用最新版本 -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
<artifactId>jts-core</artifactId>
|
||||
<version>1.18.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>com.fangjia</groupId>-->
|
||||
<!-- <artifactId>fangjia-open</artifactId>-->
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+3
-1
@@ -4209,7 +4209,7 @@ class CommonDatautils {
|
||||
|
||||
|
||||
static def findAllNewPoiDatas(city){
|
||||
String sql = "select pname,cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
|
||||
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
@@ -4222,6 +4222,8 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def findOneThirdNetWorkOffice(city, region, name){
|
||||
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
|
||||
def map = [:]
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
|
||||
class ProxyHttpUtils {
|
||||
|
||||
static String proxyHost = 'tunnel2.qg.net';
|
||||
static Integer proxyPort = 14186;
|
||||
static String proxyUsername = '113BBC06';
|
||||
static String proxyPassword = '3B2FBD05B0AF';
|
||||
|
||||
static String getProxyResult(targetUrl, data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
String jsonData = new JsonOutput().toJson(data);
|
||||
// 创建代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
|
||||
// 创建 URL 对象
|
||||
URL url = new URL(targetUrl);
|
||||
|
||||
// 创建连接对象
|
||||
URLConnection connection = url.openConnection(proxy);
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST");
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true);
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json");
|
||||
|
||||
// 发送请求参数
|
||||
OutputStream outputStream = connection.getOutputStream();
|
||||
outputStream.write(jsonData.getBytes("UTF-8"));
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 读取响应
|
||||
int responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line);
|
||||
}
|
||||
reader.close();
|
||||
return response.toString();
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.gj.coresite
|
||||
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import java.io.OutputStream
|
||||
|
||||
// 目标服务器的URL
|
||||
def targetUrlString = "http://data.zz.baidu.com/urls?site=sh.fangjia.com&token=UaKTemkcQ6I4YvSA"
|
||||
|
||||
// 需要POST的URL数据
|
||||
def postData = "https://sh.fangjia.com/zoushi?cityName=%E4%B8%8A%E6%B5%B7&keyword=%E5%A4%A9%E5%B1%B1%E6%96%B0%E8%8B%91"
|
||||
|
||||
// 创建URL对象
|
||||
URL url = new URL(targetUrlString)
|
||||
|
||||
// 打开连接
|
||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection()
|
||||
connection.setDoOutput(true) // 表明我们将发送POST请求的输出数据
|
||||
connection.setRequestMethod("POST") // 设置请求方法为POST
|
||||
connection.setRequestProperty("User-Agent", "curl/7.12.1") // 设置User-Agent,尽管这在实际应用中可能并非必需
|
||||
connection.setRequestProperty("Content-Type", "text/plain") // 设置内容类型,因为我们将发送纯文本数据
|
||||
connection.setRequestProperty("Content-Length", Integer.toString(postData.getBytes().length)) // 设置内容长度
|
||||
|
||||
// 发送POST数据
|
||||
//try (OutputStream os = connection.getOutputStream()) {
|
||||
// byte[] input = postData.getBytes("utf-8")
|
||||
// os.write(input, 0, input.length)
|
||||
//}
|
||||
outputStream = connection.getOutputStream();
|
||||
outputStream.write(postData.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 获取响应码
|
||||
int responseCode = connection.getResponseCode()
|
||||
println "POST Response Code :: $responseCode"
|
||||
|
||||
// 可选:读取响应内容
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"))
|
||||
StringBuilder response = new StringBuilder()
|
||||
String responseLine
|
||||
while ((responseLine = reader.readLine()) != null) {
|
||||
response.append(responseLine.trim())
|
||||
}
|
||||
println "Response Body :: ${response.toString()}"
|
||||
|
||||
// 关闭连接
|
||||
connection.disconnect()
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.io.FileType
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class AddDevShapeData {
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
def devAreaMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDevShapeData();
|
||||
task.run()
|
||||
// task.streetInfo()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\111开发区\\'
|
||||
def dir = new File(directoryPath)
|
||||
if (dir.isDirectory()) {
|
||||
// dir.eachFileRecurse(FileType.FILES) { file ->
|
||||
// process(file.absolutePath)
|
||||
// }
|
||||
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
|
||||
} else {
|
||||
println "The specified path is not a directory."
|
||||
}
|
||||
}
|
||||
|
||||
def addDevInfo(file){
|
||||
ExcelUtils.multiThreadingModify(file, {d->
|
||||
// if(!d['I']){
|
||||
def name = d['D']
|
||||
def q = new BasicDBObject()
|
||||
q.put('_id',new ObjectId(d['A']))
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
|
||||
if(obj){
|
||||
if(obj.operator){
|
||||
d['H'] = obj.operator
|
||||
}
|
||||
if(obj.operate_result){
|
||||
d['I'] = obj.operate_result
|
||||
}
|
||||
if(obj.m_date){
|
||||
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
// }
|
||||
devAreaMap.each {k,v->
|
||||
if(name.contains(k)){
|
||||
d['M'] = v.area_info
|
||||
d['N'] = v.count
|
||||
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def process(file){
|
||||
def retMap = getNewLocations(file)
|
||||
if(retMap && retMap.gaode.size()>0){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+retMap.name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
|
||||
if(obj){
|
||||
for(def i=0;i<retMap.gaode.size();i++){
|
||||
def udata = new BasicDBObject()
|
||||
def resultMap = [:]
|
||||
def bd_resultMap = [:]
|
||||
|
||||
resultMap.type = "polygon"
|
||||
resultMap.coordinates = [retMap.gaode[i]]
|
||||
|
||||
bd_resultMap.type = "polygon"
|
||||
bd_resultMap.coordinates = [retMap.baidu[i]]
|
||||
if(i>0){
|
||||
obj.removeField("_id")
|
||||
obj.put("create_time", new Date())
|
||||
obj.put("block", retMap.name+"-地块"+i)
|
||||
obj.put("shape", resultMap)
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("bd_shape",bd_resultMap)
|
||||
obj.put("operator", "外包")
|
||||
obj.put( "operate_result" , "已处理")
|
||||
DEVELOPMENT_ZONE_43_COLL.save(obj)
|
||||
}else{
|
||||
udata.put("shape", resultMap)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("bd_shape",bd_resultMap)
|
||||
udata.put("operate_result" , "已处理")
|
||||
udata.put("operator", "外包")
|
||||
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getNewLocations(file){
|
||||
def jsonObj = readJson(file)
|
||||
def features = jsonObj.features
|
||||
def gaode_locations = []
|
||||
def baidu_locations = []
|
||||
def name = null
|
||||
def areaMap = [:]
|
||||
if(features && features.size()==1){
|
||||
name = features[0].properties.Name
|
||||
def coordinates = features[0].geometry.coordinates
|
||||
|
||||
def index = 0
|
||||
println name +" 有 "+coordinates.size()+" 块"
|
||||
|
||||
areaMap.count = coordinates.size()
|
||||
def areas = []
|
||||
if(name.contains("泉州经济技术开发区")){
|
||||
for(def pointList : coordinates){
|
||||
index ++
|
||||
def gaode_coordinates = []
|
||||
def bd_coordinates = []
|
||||
def points = null
|
||||
if(pointList.size()>1){
|
||||
points = pointList
|
||||
}else{
|
||||
points = pointList[0]
|
||||
}
|
||||
double area = com.fangjia.script.util.GeoHelper.calculateArea(points)
|
||||
println name +" 地块"+index + "围栏面积是: ${area} 平方公里"
|
||||
areas << area
|
||||
def locationstr = ""
|
||||
for(def arrs : points){
|
||||
def gaode_map = GeoUtils.gps84ToGcj02(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
gaode_coordinates << [gaode_map.get("lng"), gaode_map.get("lat")]
|
||||
|
||||
//此高德转百度 有偏差
|
||||
def bd_map = GeoUtils.gcj02ToBd09(gaode_map.get("lat"), gaode_map.get("lng"))
|
||||
// bd_coordinates << [bd_map.get("lng"), bd_map.get("lat")]
|
||||
// 高德转百度 通过接口有次数限制
|
||||
// def baidu = BaiDuApiUtils.geoconv(gaode_map.get("lng"), gaode_map.get("lat"), 3)
|
||||
// bd_coordinates << [baidu.x, baidu.y]
|
||||
|
||||
locationstr = arrs[0].toString()+","+arrs[1].toString()
|
||||
def bdpoint = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()), 6)
|
||||
if(bdpoint){
|
||||
bd_coordinates << bdpoint
|
||||
}
|
||||
}
|
||||
//整个一串一起传超过4029会报错
|
||||
// bd_coordinates = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()-1), 6)
|
||||
gaode_locations << gaode_coordinates
|
||||
if(bd_coordinates){
|
||||
baidu_locations << bd_coordinates
|
||||
}else{
|
||||
println name +" 转换有问题"
|
||||
}
|
||||
|
||||
}
|
||||
areaMap.area_info = areas.join("\n")
|
||||
}
|
||||
|
||||
}else{
|
||||
println file +" 有多个"
|
||||
}
|
||||
|
||||
devAreaMap.put(name, areaMap)
|
||||
def retMap = [:]
|
||||
retMap.name = name
|
||||
retMap.gaode = gaode_locations
|
||||
retMap.baidu = baidu_locations
|
||||
return retMap
|
||||
}
|
||||
|
||||
def readJson(def file){
|
||||
def jsonSlurper = new JsonSlurper()
|
||||
def jsonFile = new File(file)
|
||||
def jsonText = jsonFile.text
|
||||
def jsonObj = jsonSlurper.parseText(jsonText)
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
def streetInfo(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx", {d->
|
||||
def count = dev_mapping_CONN.count(new BasicDBObject("name", d['E']).append("type", "乡镇街道"))
|
||||
d['L'] = count
|
||||
})
|
||||
}
|
||||
}
|
||||
+142
-17
@@ -23,19 +23,19 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
//补充库中没有详细地址的小区
|
||||
class AddDistrictAddress extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
// def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
// def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
//
|
||||
// DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
// public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
//
|
||||
//
|
||||
// public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
//
|
||||
// def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
// DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
//
|
||||
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictAddress(), args)
|
||||
@@ -43,7 +43,16 @@ class AddDistrictAddress extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// addNoAddressDist() //为不精确的地址通过地图补充地址
|
||||
// mapOtherAddress() //有精确地址的寻找其它地址
|
||||
|
||||
// addressSelect()
|
||||
addrUpdateToDB()
|
||||
}
|
||||
|
||||
|
||||
def addNoAddressDist(){
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def index = 0
|
||||
// def allrets = []
|
||||
// for(def c : citys){
|
||||
@@ -65,9 +74,95 @@ class AddDistrictAddress extends BaseBatch{
|
||||
// allrets.clear()
|
||||
// }
|
||||
// }
|
||||
updateDistAddr()
|
||||
}
|
||||
|
||||
updateDistAddr()
|
||||
|
||||
def mapOtherAddress(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
def allrets = []
|
||||
for(def c : citys){
|
||||
index ++
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address", "addr_alias"])
|
||||
def radInfoDists = haveRoadInfoDistricts(districts)
|
||||
println c +" "+index +" 原始小区量:"+districts.size() +" 有详细地址量:"+radInfoDists.size()
|
||||
radInfoDists = baiduMatch(c, radInfoDists)
|
||||
radInfoDists = gaodeMatch(c, radInfoDists)
|
||||
|
||||
def rets = []
|
||||
rets << ["id", "城市", "区域", "名称", "地址","地址别名", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
|
||||
for(def d : radInfoDists){
|
||||
def addr_alias_str = '';
|
||||
if(d.addr_alias){
|
||||
addr_alias_str = d.addr_alias.join(" ")
|
||||
}
|
||||
rets << [d._id.toString(), d.city, d.region, d.name, d.address, addr_alias_str,d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
|
||||
}
|
||||
allrets.addAll(rets)
|
||||
if(allrets.size()>50000){
|
||||
ExcelUtils.write("D:\\数据\\小区\\小区精确地址补充"+index+".xlsx", allrets, "补充地址")
|
||||
allrets.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addressSelect(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-all-2.xlsx",{d->
|
||||
if(d['G']){
|
||||
d['O'] = AddressUtil.formatAddress(d['B'], d['G'])
|
||||
}
|
||||
if(d['I']){
|
||||
d['P'] = AddressUtil.formatAddress(d['B'], d['I'])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addrUpdateToDB(){
|
||||
def updateDist = 0
|
||||
def addAddrCount = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-入库-2.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def oldAddrAliasCount = 0
|
||||
if(dist.addr_alias){
|
||||
oldAddrAliasCount = dist.addr_alias.size()
|
||||
}
|
||||
def new_addr_alias = null
|
||||
if(d['K'].contains("新增百度地址") || d['K'].contains("双地图新增")){
|
||||
new_addr_alias = getNewAddressAlias(d['L'], dist.address, dist.addr_alias)
|
||||
}
|
||||
if(d['K'].contains("新增高德地址") || d['K'].contains("双地图新增")){
|
||||
new_addr_alias = getNewAddressAlias(d['M'], dist.address, dist.addr_alias)
|
||||
}
|
||||
if(new_addr_alias){
|
||||
updateDist++
|
||||
println updateDist
|
||||
addAddrCount = addAddrCount + new_addr_alias.size()-oldAddrAliasCount
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("addr_alias", new_addr_alias)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
println "修改小区数:"+updateDist
|
||||
println "新增地址数:"+addAddrCount
|
||||
}
|
||||
|
||||
def getNewAddressAlias(new_addr, old_addr, dist_addr_alias){
|
||||
def arrs = new_addr.split(";")
|
||||
def addr_alias = []
|
||||
if(dist_addr_alias){
|
||||
addr_alias = dist_addr_alias
|
||||
}
|
||||
for(def address : arrs){
|
||||
if(old_addr!=address && !addr_alias.contains(address)){
|
||||
addr_alias << address
|
||||
}
|
||||
}
|
||||
return addr_alias
|
||||
}
|
||||
|
||||
def updateDistAddr(){
|
||||
@@ -108,6 +203,19 @@ class AddDistrictAddress extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def haveRoadInfoDistricts(districts){
|
||||
def rets = []
|
||||
for(def d : districts){
|
||||
if(d.address){
|
||||
def flag = DataHandleUtils.isStandardRoad(d.address)
|
||||
if(flag){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
@@ -166,7 +274,8 @@ class AddDistrictAddress extends BaseBatch{
|
||||
retMap.old_addr = temp_address
|
||||
}
|
||||
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
|
||||
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
|
||||
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
|
||||
retMap.new_addr = nineAddrSplit(fullCityName, temp_address)
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -175,6 +284,21 @@ class AddDistrictAddress extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def nineAddrSplit(city, addr){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
|
||||
def address = ""
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
|
||||
address = jsonObject.get("street")
|
||||
}
|
||||
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
|
||||
address = address + jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
return address
|
||||
}
|
||||
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
@@ -250,7 +374,8 @@ class AddDistrictAddress extends BaseBatch{
|
||||
if(temp_addr){
|
||||
retMap.old_addr = temp_addr
|
||||
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
|
||||
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
|
||||
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
|
||||
retMap.new_addr = nineAddrSplit(fullCityName, temp_addr)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
//
|
||||
// }
|
||||
|
||||
checkRangeCount()
|
||||
// checkRangeCount()
|
||||
// synch100()
|
||||
}
|
||||
|
||||
|
||||
+16
-12
@@ -84,6 +84,7 @@ class DevProcess {
|
||||
// task.addQiDevResult() //获取齐梁数据结果
|
||||
|
||||
// task.updateDevShape()//修改开发区结构
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -439,30 +440,33 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
|
||||
if(!d['B'] && d['A']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
|
||||
if(!d['E'] && d['D']){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d['A'])
|
||||
// q.put("city",d['G'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['A']]))
|
||||
def name = d['A'].replace('经济技术开发区','经济开发区')
|
||||
// q.put("name", d['D'])
|
||||
q.put("city",d['C'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['D']]))
|
||||
def name = d['D'] //.replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['I'] = "库中有开发区-相似2"
|
||||
d['G'] = "相似"
|
||||
// d['K'] = obj.name
|
||||
// d['C'] = obj._id.toString()
|
||||
d['B'] = obj.name
|
||||
// if(obj.name == d['A']){
|
||||
// d['E'] = "相同"
|
||||
d['E'] = obj.name
|
||||
// if(obj.name == d['D']){
|
||||
// d['G'] = "相同"
|
||||
// }else{
|
||||
// d['E'] = "包含"
|
||||
// d['G'] = "包含"
|
||||
// }
|
||||
if(obj.shape){
|
||||
d['F'] = "是"
|
||||
}
|
||||
// d['E'] = "别名"
|
||||
// d['F'] = obj.name
|
||||
}
|
||||
}
|
||||
},1);
|
||||
},0);
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.script.gj.ProxyHttpUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
class DevQlCheck {
|
||||
|
||||
//http://146.56.217.44:8088/uams/index
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevQlCheck()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def devName = "福建龙海经济开发区"
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",devName))
|
||||
def pois = readCSV("C:\\Users\\gaojuan\\Downloads\\漳州市POI.csv")
|
||||
def devPois = []
|
||||
devPois << ["省", "城市", "区域", "名称", "地址", "类型", "经度", "纬度", "齐梁开发区"]
|
||||
for(def poi : pois){
|
||||
def flag = InsidePolygon.isInArea(poi.lng.toString().toDouble(), poi.lat.toString().toDouble(), InsidePolygon.createPoint4(obj.shape.coordinates))
|
||||
if(flag){
|
||||
devPois << [poi.province, poi.city, poi.district, poi.name, poi.address, poi.cat, poi.lng, poi.lat]
|
||||
}
|
||||
}
|
||||
println "共:"+devPois.size()+" 个"
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx", devPois, "齐梁开发区")
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx",{d->
|
||||
def address = d['B']+d['C']+d['D']
|
||||
// if(d['E']){
|
||||
// address = address + d['E']
|
||||
// }
|
||||
Thread.sleep(3000)
|
||||
def s = getQlResult(address)
|
||||
d['I'] = s
|
||||
index ++
|
||||
println index+" "+ s
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
|
||||
def getQlResult(address){
|
||||
def params = [
|
||||
'address': address,
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
try{
|
||||
def str = ProxyHttpUtils.getProxyResult(targetUrl, params);
|
||||
println address +" "+ str
|
||||
JSONObject result = JSONObject.fromObject(str)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println address +" "+e.getMessage()
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("province", arrs[0])
|
||||
o.put("city", arrs[1])
|
||||
o.put("district", arrs[2])
|
||||
o.put("name", arrs[3])
|
||||
o.put("township", arrs[4])
|
||||
o.put("address", arrs[5])
|
||||
o.put("cat", arrs[6])
|
||||
o.put("lng", arrs[7])
|
||||
o.put("lat", arrs[8])
|
||||
o.put("data_source", arrs[9])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println filename+" 文件异常"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -32,7 +32,7 @@ class DevUpdateTestBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
task.addDevAddress(path)
|
||||
//
|
||||
@@ -167,7 +167,7 @@ class DevUpdateTestBatch {
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+140
-39
@@ -1,12 +1,14 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -28,14 +30,24 @@ class GXDdistrictMatch {
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
task.addOldMatch2()
|
||||
// task.addOldMatch2()
|
||||
// task.runDb()
|
||||
// task.hainanMatch()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
// task.runMatch()
|
||||
|
||||
// task.checkDist()
|
||||
|
||||
task.addPrice()
|
||||
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
@@ -94,6 +106,46 @@ class GXDdistrictMatch {
|
||||
|
||||
}
|
||||
|
||||
def hainanMatch(){
|
||||
def fileName = "D:\\数据\\小区\\海南未匹配地址-1.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['D'] && d['F'] ){//
|
||||
// def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['I'])
|
||||
// if(jsonObject){
|
||||
//// if(jsonObject.get("district") && jsonObject.get("district")!='null'){
|
||||
//// d['AO'] = jsonObject.get("district")
|
||||
//// }
|
||||
// if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
// d['AP'] = jsonObject.get("province")
|
||||
// }
|
||||
// if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
// d['AI'] = jsonObject.get("city")
|
||||
// }
|
||||
// }
|
||||
def formatCity = RegionData.formatCity(d['F'])
|
||||
def fullCityObj = fullCityMap.get(formatCity)
|
||||
if(fullCityObj){
|
||||
// def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
|
||||
|
||||
def obj = findDatasByCityAndName(formatCity, d['D'].replaceAll('[·\\(\\)()]|·', ''))
|
||||
if(obj){
|
||||
d['U'] = "楼盘抓取库"
|
||||
d['V'] = obj.city
|
||||
d['W'] = obj.region
|
||||
d['X'] = obj.name
|
||||
d['Y'] = obj.property_type
|
||||
d['Z'] = obj.address
|
||||
def loc = obj.location
|
||||
if(loc){
|
||||
d['AA'] = loc.x
|
||||
d['AB'] = loc.y
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def runDb(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
@@ -238,43 +290,41 @@ class GXDdistrictMatch {
|
||||
|
||||
def saveToDB(){
|
||||
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
|
||||
def fileName = "D:\\数据\\小区\\海南新增小区.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['A']){
|
||||
if(d['A'] && d['C']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def alias = []
|
||||
if(d['G'] && name!=d['G']){
|
||||
alias << d['G']
|
||||
}
|
||||
// def alias = []
|
||||
// if(d['G'] && name!=d['G']){
|
||||
// alias << d['G']
|
||||
// }
|
||||
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
// alias << d['AI']
|
||||
// }
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
// if(alias){
|
||||
// udata.put("alias", alias)
|
||||
// }
|
||||
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['E'])
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("status", 0)
|
||||
udata.put("name_cat", 0)
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
// udata.put("username", "高娟")
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
// if(d['AH']){
|
||||
// type = d['AH']
|
||||
// }else{
|
||||
// type = d['Y']
|
||||
// }
|
||||
if(d['D']){
|
||||
type = d['D']
|
||||
}
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
@@ -282,26 +332,50 @@ class GXDdistrictMatch {
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
udata.put("estate_type2", estateType2)
|
||||
|
||||
if(d['F']){
|
||||
Long lng = d['F']*10000000000
|
||||
udata.put("lng", lng)
|
||||
}
|
||||
if(d['E']){
|
||||
Long lat = d['E'] * 10000000000
|
||||
Long lat = d['G'] * 10000000000
|
||||
udata.put("lat", lat)
|
||||
}
|
||||
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
def opt = "add"
|
||||
if(s && s=='成功'){
|
||||
saveOptLog(udata.city, udata.region, udata.name, opt)
|
||||
udata.put("lng", lng)
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(d['F'].toString().toDouble(), d['G'].toString().toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
if(lng2){
|
||||
Long lng2_new = lng2 *10000000000L
|
||||
Long lat2_new = lat2 *10000000000L
|
||||
udata.put("lng2", lng2_new)
|
||||
udata.put("lat2", lat2_new)
|
||||
}
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map",markMap)
|
||||
// udata.put("is_new", 0)
|
||||
DbUtils.seaweed.save(udata)
|
||||
saveOptLog(udata.city, udata.region, udata.name, "add")
|
||||
// def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println name
|
||||
// def opt = "add"
|
||||
// if(s && s=='成功'){
|
||||
// saveOptLog(udata.city, udata.region, udata.name, opt)
|
||||
// }
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def checkDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\海南未匹配地址-1.xlsx", { d ->
|
||||
if(d['V'] && d['W'] && d['X']){
|
||||
matching(d['V'], d['W'], d, d['X'], null, null)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(city, region, d){
|
||||
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
|
||||
@@ -391,7 +465,7 @@ class GXDdistrictMatch {
|
||||
if(rets && rets.size()>=1){
|
||||
d = setRowValue(rets[0], d, "名称在地址中"+str)
|
||||
}else{
|
||||
if (d['D']) {
|
||||
if (d['D'] && alias) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
@@ -449,14 +523,14 @@ class GXDdistrictMatch {
|
||||
|
||||
def setRowValue(dbo, d, remark){
|
||||
if (dbo) {
|
||||
d['V'] = dbo.city
|
||||
d['W'] = dbo.region
|
||||
d['X'] = dbo.name
|
||||
d['Y'] = dbo.estate_type2
|
||||
d['Z'] = dbo.status
|
||||
d['AA'] = dbo.name_cat
|
||||
d['U'] = remark
|
||||
d['AH'] = dbo.address
|
||||
// d['V'] = dbo.city
|
||||
// d['W'] = dbo.region
|
||||
// d['X'] = dbo.name
|
||||
// d['Y'] = dbo.estate_type2
|
||||
d['AK'] = dbo.status
|
||||
// d['AA'] = dbo.name_cat
|
||||
d['AJ'] = remark
|
||||
// d['AH'] = dbo.address
|
||||
}
|
||||
return d;
|
||||
}
|
||||
@@ -510,6 +584,33 @@ class GXDdistrictMatch {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def findDatasByCityAndName(def city, def name){
|
||||
def sql = "select * from community where city='${city}' and name='${name}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
println d.property_type
|
||||
})
|
||||
|
||||
if(rets){
|
||||
def obj = null
|
||||
for(def r : rets){
|
||||
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
|
||||
obj = r
|
||||
break
|
||||
}
|
||||
}
|
||||
if(obj){
|
||||
return obj;
|
||||
}else{
|
||||
return rets[0];
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, type){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
|
||||
+3
-2
@@ -467,7 +467,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def map = new HashMap()
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "map_poi_2024", queryBuilder, { String id, Map d->
|
||||
def district = ""
|
||||
def district = "嘉峪关市"
|
||||
if(d.district){
|
||||
district = d.district
|
||||
}
|
||||
@@ -477,11 +477,12 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("district", district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
regionMap.put(d.name.replace("-","").replace("¡¤",""), o)
|
||||
// map.put(district, regionMap)
|
||||
map.put(district, regionMap)
|
||||
})
|
||||
return map
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class PoiNineSplitBatch{
|
||||
|
||||
static main(args) {
|
||||
public static void main(args) {
|
||||
def task = new PoiNineSplitBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
+225
-30
@@ -3,7 +3,9 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
@@ -12,6 +14,13 @@ import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
@@ -22,6 +31,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def updateList = []
|
||||
def addAliasList = []
|
||||
@@ -38,7 +48,8 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def TownStandard(){
|
||||
// townStandardByCat("office")
|
||||
townStandardByPoi()
|
||||
// townStandardByPoi()
|
||||
addTownByTownShape()
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
|
||||
@@ -69,11 +80,15 @@ class TownStandardBatch extends BaseBatch{
|
||||
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||
|
||||
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
|
||||
//浙江,江苏,安徽
|
||||
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
|
||||
def runProvinces = ["福建省","浙江省","江苏省","安徽省","山东省","甘肃省"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
|
||||
index++
|
||||
def data = fullCityMap.get(city)
|
||||
// if(runProvinces.contains(data.province)){
|
||||
// continue
|
||||
// }
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
@@ -95,11 +110,16 @@ class TownStandardBatch extends BaseBatch{
|
||||
def rets = readPoiCsv(fileName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
if(updateList.size()>50000){
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-${index}.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名-${index}.xlsx", addAliasList, "数据")
|
||||
updateList.clear()
|
||||
}
|
||||
}else{
|
||||
println fileName+" 文件没找到"
|
||||
}
|
||||
@@ -107,8 +127,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-最后.xlsx", updateList, "数据")
|
||||
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
@@ -168,7 +187,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
for(def d : rets){
|
||||
@@ -221,21 +240,21 @@ class TownStandardBatch extends BaseBatch{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":updateTown])
|
||||
results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
updateList << [d.id, d.province, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":""])
|
||||
// results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
updateList << [d.id, d.province, d.city, d.district, d.name, d.township, d.status, ""]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
|
||||
}
|
||||
}
|
||||
@@ -244,10 +263,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
// if(results.size()>0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
if(results.size()>0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println fullCityName +" 处理完成"
|
||||
}
|
||||
|
||||
@@ -322,18 +341,194 @@ class TownStandardBatch extends BaseBatch{
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
try{
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def readCSV2(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
def index = 0
|
||||
for(line in lines) {
|
||||
if(line && index>=1){
|
||||
String[] arrs = line.split(",")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("id", arrs[1])
|
||||
o.put("province", arrs[2])
|
||||
o.put("city", arrs[3])
|
||||
o.put("district", arrs[4])
|
||||
o.put("lng", arrs[5])
|
||||
o.put("lat", arrs[6])
|
||||
o.put("data_source", arrs[7])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
}
|
||||
index++
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addTownByTownShape(){
|
||||
def i = 0
|
||||
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式"]
|
||||
while(i<50){
|
||||
def fileName = "/home/goojia/gj/new_poi/input/no_township_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()) {
|
||||
def rets = readCSV2(fileName)
|
||||
def rows = addTownshipBatch(rets)
|
||||
if (CommonDatautils.isWindow()) {
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}else{
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}
|
||||
println fileName +" 运行完成"
|
||||
}else{
|
||||
println fileName +" 不存在"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
def addTownshipBatch(def rets){
|
||||
def results = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def r = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def rMap = regionFullNameMap.get(r.city)
|
||||
if (rMap) {
|
||||
def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
|
||||
def set = rMap.keySet()
|
||||
println totalCount.get() +" "+r.city + " " + set.size()
|
||||
if (set.size() >= 1) {
|
||||
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
|
||||
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source]
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return results
|
||||
}
|
||||
|
||||
def addTown(saveObj, townMap, location) {
|
||||
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
|
||||
def towns = townMap.get(saveObj.district)
|
||||
if (towns) {
|
||||
for (def town : towns) {
|
||||
if (town.shape && town.shape.coordinates) {
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if (flag) {
|
||||
saveObj.township = town.street
|
||||
saveObj.town_source = "同区域"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
|
||||
if(!saveObj.township){
|
||||
def townNames = townMap.keySet()
|
||||
for(def townName : townNames){
|
||||
towns = townMap.get(townName)
|
||||
for (def town : towns) {
|
||||
if (town.shape && town.shape.coordinates) {
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if (flag) {
|
||||
saveObj.township = town.street
|
||||
saveObj.new_district = town.mzb_district_name
|
||||
saveObj.town_source = "忽略区域"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if(saveObj.township){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
//通过调取地图api接口获取乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// saveObj.township = gaodeMap.township
|
||||
// saveObj.new_district = gaodeMap.district
|
||||
// saveObj.town_source = "高德API"
|
||||
// }
|
||||
// }
|
||||
return saveObj
|
||||
}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
city = addressComponent.city
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+4
-4
@@ -38,16 +38,16 @@ class TownshipStrandUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
// task.process()
|
||||
// task.updateTown()
|
||||
task.process()
|
||||
task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
task.updateDiffRegionTown()
|
||||
// task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
// task.updateDevMapping()
|
||||
println new Date().getTime()
|
||||
// println new Date().getTime()
|
||||
// task.updateSpecCityTown()
|
||||
}
|
||||
|
||||
|
||||
+42
-1
@@ -67,7 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictFieldAddBatch(), args)
|
||||
// runBatch(new DistrictFieldAddBatch(), args)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
|
||||
// if(dist){
|
||||
// d['L'] = dist.loudong_count
|
||||
// d['M'] = dist.household_count
|
||||
// d['Q'] = dist.estate_type2
|
||||
// }
|
||||
def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
|
||||
if(!list && dist){
|
||||
def alias = dist.alias
|
||||
for(def name : alias){
|
||||
list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
|
||||
if(list){
|
||||
d['Z'] = "别名"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['Z'] = "名称"
|
||||
}
|
||||
if(list){
|
||||
def obj = list[0]
|
||||
if(obj.loudong_cnt){
|
||||
d['W'] = obj.loudong_cnt
|
||||
}
|
||||
|
||||
if(obj.households){
|
||||
d['X'] = obj.households
|
||||
}
|
||||
d['Y'] = obj.url
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -361,6 +393,15 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
return list
|
||||
}
|
||||
|
||||
|
||||
static def findEsDatasByCityAndName(c, district, name, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' and region='${district}' and name='${name}'", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
|
||||
+28
@@ -398,4 +398,32 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 经纬度转换
|
||||
* 1、高德转百度 2、高德转WGS84 3、百度转高德 4、WGS84转高德 5、百度转WGS84 6、WGS84转百度
|
||||
*/
|
||||
public static def convert(String locationstr, type){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
def points = []
|
||||
try {
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/convert?locations="+URLEncoder.encode(locationstr,"UTF-8")+"&coordsys=${type}"
|
||||
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
if(jsonObj && jsonObj.locations){
|
||||
def locations = jsonObj.locations
|
||||
for(def s : locations){
|
||||
def arrs = s.split(",")
|
||||
points = [arrs[0].toString().toDouble(), arrs[1].toString().toDouble()]
|
||||
}
|
||||
}
|
||||
return points
|
||||
} catch (Exception e) {
|
||||
println e.getMessage()
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+14
-14
@@ -13,18 +13,18 @@ class MsbPocTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\房讯通10月价格核查_1126-FXT提交结果20241205.xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
// matchOld(inputfile)
|
||||
matchOld(inputfile)
|
||||
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
|
||||
// test("D:\\数据\\美术宝\\test2.xlsx")
|
||||
|
||||
// cityPrice()
|
||||
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
|
||||
|
||||
addCCBPrice(inputfile)
|
||||
// addCCBPrice(inputfile)
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
@@ -69,24 +69,24 @@ class MsbPocTest {
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Y'] && row['G']){
|
||||
if(!row['AD'] ){
|
||||
def city = row['C']
|
||||
def region = null //row['D']
|
||||
def name = row['E']
|
||||
def address = row['G']
|
||||
def name = null// row['E']
|
||||
def address = row['D']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
row['AC'] = jsonObject.get("propertyType")
|
||||
row['AD'] = jsonObject.get("id")
|
||||
row['AE'] = jsonObject.get("city")
|
||||
row['AF'] = jsonObject.get("district")
|
||||
row['AG'] = jsonObject.get("name")
|
||||
row['AH'] = jsonObject.get("propertyType")
|
||||
// row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
@@ -114,7 +114,7 @@ class MsbPocTest {
|
||||
if(!row['Y']){
|
||||
def city = row['C']
|
||||
def region = null //row['B']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
def address = row['D'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
@@ -125,7 +125,7 @@ class MsbPocTest {
|
||||
row['AB'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+23
-21
@@ -38,9 +38,9 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
def filePath = "D:\\Êý¾Ý\\¼Û¸ñ\\»ÝÖÝ.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
@@ -174,26 +174,28 @@ class AddPriceTest {
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['U']){
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(d['A']){
|
||||
def cityId = cityCodeDataMap.get(d['B'])
|
||||
def map = findAvgPrice(d['A'], cityId)
|
||||
if(map){
|
||||
d['H'] = map.avg_price
|
||||
d['I'] = map.calc_type
|
||||
d['I'] = map.avg_price
|
||||
d['J'] = map.month
|
||||
// d['H'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
|
||||
// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId)
|
||||
// if(newPriceMap){
|
||||
// d['Q'] = newPriceMap.avg_price
|
||||
// }
|
||||
def newPriceMap = findNewFjwAvgPrice(d['A'], cityId)
|
||||
if(newPriceMap){
|
||||
d['G'] = newPriceMap.avg_price
|
||||
d['H'] = newPriceMap.month
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -213,7 +215,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findNewFjwAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance235.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -270,14 +272,14 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['B']) {
|
||||
if (!d['H']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
if(gxdPrice){
|
||||
d['B'] = gxdPrice.price
|
||||
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
|
||||
d['H'] = gxdPrice.price
|
||||
d['I'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -305,11 +307,11 @@ class AddPriceTest {
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
// if(!d['L']){
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['AJ'])
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
d['E'] = dist.name
|
||||
// d['E'] = dist.name
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.name_cat
|
||||
// d['F'] = dist.estate_type2
|
||||
@@ -318,9 +320,9 @@ class AddPriceTest {
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['G'] = dbo.avg_price
|
||||
d['E'] = dbo.avg_price_2b
|
||||
// d['X'] = dbo.cal_type
|
||||
// d['E'] = dbo.avg_price_2b
|
||||
// d['E'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
// }
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
|
||||
import java.text.DecimalFormat
|
||||
|
||||
class GeoHelper {
|
||||
|
||||
|
||||
@@ -60,10 +64,159 @@ class GeoHelper {
|
||||
// return GetLatLon(a, distance, angle);
|
||||
// }
|
||||
//
|
||||
public static void main(String[] args){
|
||||
GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
|
||||
GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
|
||||
}
|
||||
public static haversine(lat1, lon1, lat2, lon2) {
|
||||
def R = 6371; // 地球半径(公里)
|
||||
def lat1Rad = lat1 * (Math.PI / 180);
|
||||
def lat2Rad = lat2 * (Math.PI / 180);
|
||||
def deltaLat = (lat2 - lat1) * (Math.PI / 180);
|
||||
def deltaLon = (lon2 - lon1) * (Math.PI / 180);
|
||||
|
||||
def a = Math.sin(deltaLat / 2) ** 2 +
|
||||
Math.cos(lat1Rad) * Math.cos(lat2Rad) *
|
||||
Math.sin(deltaLon / 2) ** 2;
|
||||
|
||||
def c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
||||
return R * c; // 返回距离(公里)
|
||||
}
|
||||
|
||||
public static calculateArea(coords) {
|
||||
def area = 0;
|
||||
def n = coords.size();
|
||||
|
||||
for (def i = 0; i < n; i++) {
|
||||
def j = (i + 1) % n;
|
||||
area += coords[i][0] * coords[j][1];
|
||||
area -= coords[j][0] * coords[i][1];
|
||||
}
|
||||
area = Math.abs(area) / 2; // 平方度
|
||||
|
||||
// 转换为平方公里
|
||||
def conversionFactor = 111.32; // 大约每度经纬度的公里数(考虑到地球的曲率)
|
||||
|
||||
def df = new DecimalFormat("#.000000")
|
||||
return Double.parseDouble(df.format(area * conversionFactor * conversionFactor)) // 返回面积(平方公里)
|
||||
// return area * conversionFactor * conversionFactor
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args){
|
||||
// GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
|
||||
// GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
|
||||
|
||||
List<List<Double>> coordinates = [
|
||||
[
|
||||
117.61878540582734,
|
||||
24.667331307258223
|
||||
],
|
||||
[
|
||||
117.61400840589488,
|
||||
24.670008244935104
|
||||
],
|
||||
[
|
||||
117.60936140565889,
|
||||
24.671837202755682
|
||||
],
|
||||
[
|
||||
117.60834440585768,
|
||||
24.668667276729821
|
||||
],
|
||||
[
|
||||
117.60696440583342,
|
||||
24.667256309201207
|
||||
],
|
||||
[
|
||||
117.60517040553236,
|
||||
24.667762297657362
|
||||
],
|
||||
[
|
||||
117.60272340547806,
|
||||
24.660100475272998
|
||||
],
|
||||
[
|
||||
117.60254340555321,
|
||||
24.658630509428463
|
||||
],
|
||||
[
|
||||
117.60206940590035,
|
||||
24.654781598282568
|
||||
],
|
||||
[
|
||||
117.60254840557604,
|
||||
24.652731645907636
|
||||
],
|
||||
[
|
||||
117.60346440599301,
|
||||
24.651555673447035
|
||||
],
|
||||
[
|
||||
117.6070314057805,
|
||||
24.652578649754957
|
||||
],
|
||||
[
|
||||
117.61605140571805,
|
||||
24.651939664372833
|
||||
],
|
||||
[
|
||||
117.6156454058372,
|
||||
24.654165613124405
|
||||
],
|
||||
[
|
||||
117.61559340595863,
|
||||
24.655023593107103
|
||||
],
|
||||
[
|
||||
117.61535440558418,
|
||||
24.658990501065318
|
||||
],
|
||||
[
|
||||
117.61662340528017,
|
||||
24.659352492642654
|
||||
],
|
||||
[
|
||||
117.61678740531192,
|
||||
24.661365445721003
|
||||
],
|
||||
[
|
||||
117.61472940542001,
|
||||
24.661623439403797
|
||||
],
|
||||
[
|
||||
117.61462940586114,
|
||||
24.661837434759093
|
||||
],
|
||||
[
|
||||
117.61470140583106,
|
||||
24.662665415879701
|
||||
],
|
||||
[
|
||||
117.61487240553515,
|
||||
24.663165403873506
|
||||
],
|
||||
[
|
||||
117.61522940591078,
|
||||
24.663494396452791
|
||||
],
|
||||
[
|
||||
117.6157014059138,
|
||||
24.663523395394918
|
||||
],
|
||||
[
|
||||
117.61627240565099,
|
||||
24.663438397829282
|
||||
],
|
||||
[
|
||||
117.61693540544979,
|
||||
24.663181403867725
|
||||
],
|
||||
[
|
||||
117.61878540582734,
|
||||
24.667331307258223
|
||||
]
|
||||
]
|
||||
|
||||
double area = calculateArea(coordinates)
|
||||
println "The area of the polygon is: ${String.format("%.2f", area)} square kilometers"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //±¦½£
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //±¦½£:c6fcd4bfbc40d73ee762b8115dca3390
|
||||
//15021384995£ºdfc2245bac8a7b2d5b62e060e59d4eca
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user