diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java index bcb042e67..059a2bcb9 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java @@ -3,6 +3,7 @@ package com.fangjia.core.util; import java.util.HashMap; import java.util.Map; + public class GeoUtils { public static final String LAT ="lat"; @@ -258,6 +259,8 @@ public class GeoUtils { return ret; } + + public static void main(String[] args){ Map location = gps84ToGcj02(31.218024629574998,121.36562884455); System.out.println(location.toString()); diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index e639f76e3..2b61cb696 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -537,6 +537,12 @@ 1.70 + + org.locationtech.jts + jts-core + 1.18.2 + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 4473e1e49..dfd784e71 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -8,7 +8,7 @@ import com.mongodb.DBCollection /** * ÉîÛÚ¹«Öںųɽ»Êý¾Ý¼Èë */ -def datas = ExcelUtils.read("D:\\Êý¾Ý\\2024Äê10ÔÂÉîÛÚ¶þÊÖ·¿³É½».xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\³É½»\\2024Äê11ÔÂÉîÛÚ¶þÊÖ·¿³É½».xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8e7863c2e..d77e63373 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4209,7 +4209,7 @@ class CommonDatautils { static def findAllNewPoiDatas(city){ - String sql = "select pname,cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'" + String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'" def rets = [] def index= 0 ElasticUtils.executeSqlQuery(sql, { map-> @@ -4222,6 +4222,8 @@ class CommonDatautils { } + + static def findOneThirdNetWorkOffice(city, region, name){ String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'" def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ProxyHttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ProxyHttpUtils.groovy new file mode 100644 index 000000000..72495191d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ProxyHttpUtils.groovy @@ -0,0 +1,62 @@ +package com.fangjia.script.gj + +import groovy.json.JsonOutput + +class ProxyHttpUtils { + + static String proxyHost = 'tunnel2.qg.net'; + static Integer proxyPort = 14186; + static String proxyUsername = '113BBC06'; + static String proxyPassword = '3B2FBD05B0AF'; + + static String getProxyResult(targetUrl, data){ + // ½« JSON ¶ÔÏóת»»Îª×Ö·û´® + String jsonData = new JsonOutput().toJson(data); + // ´´½¨´úÀí¶ÔÏó + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + + // ´´½¨ URL ¶ÔÏó + URL url = new URL(targetUrl); + + // ´´½¨Á¬½Ó¶ÔÏó + URLConnection connection = url.openConnection(proxy); + + // ÉèÖôúÀíÈÏÖ¤ + java.net.Authenticator.setDefault(new java.net.Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()) + } + }) + + // ÉèÖÃÇëÇó·½·¨Îª POST + connection.setRequestMethod("POST"); + + // ÔÊÐíÊä³öÁ÷ + connection.setDoOutput(true); + + // ÉèÖÃÇëÇóÍ· + connection.setRequestProperty("Content-Type", "application/json"); + + // ·¢ËÍÇëÇó²ÎÊý + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(jsonData.getBytes("UTF-8")); + outputStream.flush(); + outputStream.close(); + + // ¶ÁÈ¡ÏìÓ¦ + int responseCode = connection.getResponseCode() + if (responseCode == HttpURLConnection.HTTP_OK) { + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + return response.toString(); + } else { + println "HTTP request failed with response code ${responseCode}"; + } + return null; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/coresite/AddUrlToBaiduSpider.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/coresite/AddUrlToBaiduSpider.groovy new file mode 100644 index 000000000..2b03fd2ae --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/coresite/AddUrlToBaiduSpider.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.gj.coresite + +import java.net.HttpURLConnection +import java.net.URL +import java.io.OutputStream + +// Ä¿±ê·þÎñÆ÷µÄURL +def targetUrlString = "http://data.zz.baidu.com/urls?site=sh.fangjia.com&token=UaKTemkcQ6I4YvSA" + +// ÐèÒªPOSTµÄURLÊý¾Ý +def postData = "https://sh.fangjia.com/zoushi?cityName=%E4%B8%8A%E6%B5%B7&keyword=%E5%A4%A9%E5%B1%B1%E6%96%B0%E8%8B%91" + +// ´´½¨URL¶ÔÏó +URL url = new URL(targetUrlString) + +// ´ò¿ªÁ¬½Ó +HttpURLConnection connection = (HttpURLConnection) url.openConnection() +connection.setDoOutput(true) // ±íÃ÷ÎÒÃǽ«·¢ËÍPOSTÇëÇóµÄÊä³öÊý¾Ý +connection.setRequestMethod("POST") // ÉèÖÃÇëÇó·½·¨ÎªPOST +connection.setRequestProperty("User-Agent", "curl/7.12.1") // ÉèÖÃUser-Agent£¬¾¡¹ÜÕâÔÚʵ¼ÊÓ¦ÓÃÖпÉÄܲ¢·Ç±ØÐè +connection.setRequestProperty("Content-Type", "text/plain") // ÉèÖÃÄÚÈÝÀàÐÍ£¬ÒòΪÎÒÃǽ«·¢ËÍ´¿Îı¾Êý¾Ý +connection.setRequestProperty("Content-Length", Integer.toString(postData.getBytes().length)) // ÉèÖÃÄÚÈݳ¤¶È + +// ·¢ËÍPOSTÊý¾Ý +//try (OutputStream os = connection.getOutputStream()) { +// byte[] input = postData.getBytes("utf-8") +// os.write(input, 0, input.length) +//} +outputStream = connection.getOutputStream(); +outputStream.write(postData.getBytes()); +outputStream.flush(); +outputStream.close(); + +// »ñÈ¡ÏìÓ¦Âë +int responseCode = connection.getResponseCode() +println "POST Response Code :: $responseCode" + +// ¿ÉÑ¡£º¶ÁÈ¡ÏìÓ¦ÄÚÈÝ +def reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8")) +StringBuilder response = new StringBuilder() +String responseLine +while ((responseLine = reader.readLine()) != null) { + response.append(responseLine.trim()) +} +println "Response Body :: ${response.toString()}" + +// ¹Ø±ÕÁ¬½Ó +connection.disconnect() \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy new file mode 100644 index 000000000..4df41ffc4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy @@ -0,0 +1,202 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.ExcelUtil +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.json.JsonSlurper +import groovy.io.FileType +import org.bson.types.ObjectId + +import java.util.regex.Pattern + +class AddDevShapeData { + + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + + def devAreaMap = [:] + + static main(args) { + def task = new AddDevShapeData(); + task.run() +// task.streetInfo() + } + + def run(){ + def directoryPath = 'D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\111¿ª·¢Çø\\' + def dir = new File(directoryPath) + if (dir.isDirectory()) { +// dir.eachFileRecurse(FileType.FILES) { file -> +// process(file.absolutePath) +// } + addDevInfo("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\ËùÓпª·¢Çø.xlsx") + } else { + println "The specified path is not a directory." + } + } + + def addDevInfo(file){ + ExcelUtils.multiThreadingModify(file, {d-> +// if(!d['I']){ + def name = d['D'] + def q = new BasicDBObject() + q.put('_id',new ObjectId(d['A'])) + def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q) + if(obj){ + if(obj.operator){ + d['H'] = obj.operator + } + if(obj.operate_result){ + d['I'] = obj.operate_result + } + if(obj.m_date){ + d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd") + } + } +// } + devAreaMap.each {k,v-> + if(name.contains(k)){ + d['M'] = v.area_info + d['N'] = v.count + d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + } + } + }) + } + + def process(file){ + def retMap = getNewLocations(file) + if(retMap && retMap.gaode.size()>0){ + def q = new BasicDBObject() + q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+retMap.name+'.*$', Pattern.CASE_INSENSITIVE))) + def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q) + if(obj){ + for(def i=0;i0){ + obj.removeField("_id") + obj.put("create_time", new Date()) + obj.put("block", retMap.name+"-µØ¿é"+i) + obj.put("shape", resultMap) + obj.put("m_date", new Date()) + obj.put("bd_shape",bd_resultMap) + obj.put("operator", "Íâ°ü") + obj.put( "operate_result" , "ÒÑ´¦Àí") + DEVELOPMENT_ZONE_43_COLL.save(obj) + }else{ + udata.put("shape", resultMap) + udata.put("m_date", new Date()) + udata.put("bd_shape",bd_resultMap) + udata.put("operate_result" , "ÒÑ´¦Àí") + udata.put("operator", "Íâ°ü") + DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false) + } + } + } + } + } + + def getNewLocations(file){ + def jsonObj = readJson(file) + def features = jsonObj.features + def gaode_locations = [] + def baidu_locations = [] + def name = null + def areaMap = [:] + if(features && features.size()==1){ + name = features[0].properties.Name + def coordinates = features[0].geometry.coordinates + + def index = 0 + println name +" ÓÐ "+coordinates.size()+" ¿é" + + areaMap.count = coordinates.size() + def areas = [] + if(name.contains("ȪÖݾ­¼Ã¼¼Êõ¿ª·¢Çø")){ + for(def pointList : coordinates){ + index ++ + def gaode_coordinates = [] + def bd_coordinates = [] + def points = null + if(pointList.size()>1){ + points = pointList + }else{ + points = pointList[0] + } + double area = com.fangjia.script.util.GeoHelper.calculateArea(points) + println name +" µØ¿é"+index + "ΧÀ¸Ãæ»ýÊÇ: ${area} ƽ·½¹«Àï" + areas << area + def locationstr = "" + for(def arrs : points){ + def gaode_map = GeoUtils.gps84ToGcj02(arrs[1].toString().toDouble(), arrs[0].toString().toDouble()) + gaode_coordinates << [gaode_map.get("lng"), gaode_map.get("lat")] + + //´Ë¸ßµÂת°Ù¶È ÓÐÆ«²î + def bd_map = GeoUtils.gcj02ToBd09(gaode_map.get("lat"), gaode_map.get("lng")) +// bd_coordinates << [bd_map.get("lng"), bd_map.get("lat")] + // ¸ßµÂת°Ù¶È ͨ¹ý½Ó¿ÚÓдÎÊýÏÞÖÆ +// def baidu = BaiDuApiUtils.geoconv(gaode_map.get("lng"), gaode_map.get("lat"), 3) +// bd_coordinates << [baidu.x, baidu.y] + + locationstr = arrs[0].toString()+","+arrs[1].toString() + def bdpoint = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()), 6) + if(bdpoint){ + bd_coordinates << bdpoint + } + } + //Õû¸öÒ»´®Ò»Æð´«³¬¹ý4029»á±¨´í +// bd_coordinates = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()-1), 6) + gaode_locations << gaode_coordinates + if(bd_coordinates){ + baidu_locations << bd_coordinates + }else{ + println name +" ת»»ÓÐÎÊÌâ" + } + + } + areaMap.area_info = areas.join("\n") + } + + }else{ + println file +" Óжà¸ö" + } + + devAreaMap.put(name, areaMap) + def retMap = [:] + retMap.name = name + retMap.gaode = gaode_locations + retMap.baidu = baidu_locations + return retMap + } + + def readJson(def file){ + def jsonSlurper = new JsonSlurper() + def jsonFile = new File(file) + def jsonText = jsonFile.text + def jsonObj = jsonSlurper.parseText(jsonText) + return jsonObj + } + + def streetInfo(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\ËùÓпª·¢Çø.xlsx", {d-> + def count = dev_mapping_CONN.count(new BasicDBObject("name", d['E']).append("type", "ÏçÕò½ÖµÀ")) + d['L'] = count + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy index d85eefdca..977423309 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictAddress.groovy @@ -23,19 +23,19 @@ import java.util.concurrent.atomic.AtomicLong //²¹³ä¿âÖÐûÓÐÏêϸµØÖ·µÄÐ¡Çø class AddDistrictAddress extends BaseBatch{ - def fullCityMap = CommonDatautils.queryFullNameCity() - def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() - - DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) - public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") - - - public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") - - def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL) - - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) +// def fullCityMap = CommonDatautils.queryFullNameCity() +// def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() +// +// DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) +// public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") +// +// +// public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") +// +// def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +// DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL) +// +// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static main(args) { runBatch(new AddDistrictAddress(), args) @@ -43,7 +43,16 @@ class AddDistrictAddress extends BaseBatch{ @Override public void process(Object args) { -// def citys = CommonDatautils.findAllCity() +// addNoAddressDist() //Ϊ²»¾«È·µÄµØÖ·Í¨¹ýµØÍ¼²¹³äµØÖ· +// mapOtherAddress() //Óо«È·µØÖ·µÄѰÕÒÆäËüµØÖ· + +// addressSelect() + addrUpdateToDB() + } + + + def addNoAddressDist(){ + // def citys = CommonDatautils.findAllCity() // def index = 0 // def allrets = [] // for(def c : citys){ @@ -65,9 +74,95 @@ class AddDistrictAddress extends BaseBatch{ // allrets.clear() // } // } + updateDistAddr() + } - updateDistAddr() + def mapOtherAddress(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + def allrets = [] + for(def c : citys){ + index ++ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address", "addr_alias"]) + def radInfoDists = haveRoadInfoDistricts(districts) + println c +" "+index +" Ô­Ê¼Ð¡ÇøÁ¿£º"+districts.size() +" ÓÐÏêϸµØÖ·Á¿£º"+radInfoDists.size() + radInfoDists = baiduMatch(c, radInfoDists) + radInfoDists = gaodeMatch(c, radInfoDists) + + def rets = [] + rets << ["id", "³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "µØÖ·","µØÖ·±ðÃû", "°Ù¶ÈÔ­µØÖ·", "°Ù¶È±ê×¼µØÖ·", "¸ßµÂÔ­µØÖ·", "¸ßµÂ±ê×¼µØÖ·"] + for(def d : radInfoDists){ + def addr_alias_str = ''; + if(d.addr_alias){ + addr_alias_str = d.addr_alias.join(" ") + } + rets << [d._id.toString(), d.city, d.region, d.name, d.address, addr_alias_str,d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""] + } + allrets.addAll(rets) + if(allrets.size()>50000){ + ExcelUtils.write("D:\\Êý¾Ý\\Ð¡Çø\\Ð¡Çø¾«È·µØÖ·²¹³ä"+index+".xlsx", allrets, "²¹³äµØÖ·") + allrets.clear() + } + } + } + + def addressSelect(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡Çø\\¾«È·µØÖ·²¹³ä-all-2.xlsx",{d-> + if(d['G']){ + d['O'] = AddressUtil.formatAddress(d['B'], d['G']) + } + if(d['I']){ + d['P'] = AddressUtil.formatAddress(d['B'], d['I']) + } + }) + } + + def addrUpdateToDB(){ + def updateDist = 0 + def addAddrCount = 0 + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡Çø\\¾«È·µØÖ·²¹³ä-Èë¿â-2.xlsx",{d-> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def oldAddrAliasCount = 0 + if(dist.addr_alias){ + oldAddrAliasCount = dist.addr_alias.size() + } + def new_addr_alias = null + if(d['K'].contains("ÐÂÔö°Ù¶ÈµØÖ·") || d['K'].contains("Ë«µØÍ¼ÐÂÔö")){ + new_addr_alias = getNewAddressAlias(d['L'], dist.address, dist.addr_alias) + } + if(d['K'].contains("ÐÂÔö¸ßµÂµØÖ·") || d['K'].contains("Ë«µØÍ¼ÐÂÔö")){ + new_addr_alias = getNewAddressAlias(d['M'], dist.address, dist.addr_alias) + } + if(new_addr_alias){ + updateDist++ + println updateDist + addAddrCount = addAddrCount + new_addr_alias.size()-oldAddrAliasCount + + def udata = new BasicDBObject() + udata.put("addr_alias", new_addr_alias) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata)) + } + } + }) + println "ÐÞ¸ÄÐ¡ÇøÊý£º"+updateDist + println "ÐÂÔöµØÖ·Êý£º"+addAddrCount + } + + def getNewAddressAlias(new_addr, old_addr, dist_addr_alias){ + def arrs = new_addr.split(";") + def addr_alias = [] + if(dist_addr_alias){ + addr_alias = dist_addr_alias + } + for(def address : arrs){ + if(old_addr!=address && !addr_alias.contains(address)){ + addr_alias << address + } + } + return addr_alias } def updateDistAddr(){ @@ -108,6 +203,19 @@ class AddDistrictAddress extends BaseBatch{ return rets; } + def haveRoadInfoDistricts(districts){ + def rets = [] + for(def d : districts){ + if(d.address){ + def flag = DataHandleUtils.isStandardRoad(d.address) + if(flag){ + rets << d + } + } + } + return rets; + } + def baiduMatch(city,districts){ def data = fullCityMap.get(city) def fullCityName = null @@ -166,7 +274,8 @@ class AddDistrictAddress extends BaseBatch{ retMap.old_addr = temp_address } if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){ - retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address) +// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address) + retMap.new_addr = nineAddrSplit(fullCityName, temp_address) continue } } @@ -175,6 +284,21 @@ class AddDistrictAddress extends BaseBatch{ } + def nineAddrSplit(city, addr){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr) + def address = "" + if (jsonObject) { + if (jsonObject.get("street") && jsonObject.get("street") != "null") { + address = jsonObject.get("street") + } + if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") { + address = address + jsonObject.get("streetNumber") + } + } + return address + } + + def findBaiduDatas(city, district, name){ def q = new BasicDBObject() q.put("city", city) @@ -250,7 +374,8 @@ class AddDistrictAddress extends BaseBatch{ if(temp_addr){ retMap.old_addr = temp_addr if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){ - retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr ) +// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr ) + retMap.new_addr = nineAddrSplit(fullCityName, temp_addr) continue } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy index 72d64ea09..796237eb0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -57,7 +57,7 @@ class AddDistrictShapeBatch extends BaseBatch{ // // } - checkRangeCount() +// checkRangeCount() // synch100() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 521be8be2..a6503133c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -84,6 +84,7 @@ class DevProcess { // task.addQiDevResult() //»ñÈ¡ÆëÁºÊý¾Ý½á¹û // task.updateDevShape()//Ð޸Ŀª·¢Çø½á¹¹ + } @@ -439,30 +440,33 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\ÐÂÅÜÊ¡·Ý\\½­ËÕÊ¡\\½­ËÕÊ¡-ÆëÁº´ýÅܵØÖ·.xlsx",{d-> - if(!d['B'] && d['A']){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\¸ßÐÂÇø.xls",{d-> + if(!d['E'] && d['D']){ def q = new BasicDBObject() -// q.put("name", d['A']) -// q.put("city",d['G']) -// q.put('alias', new BasicDBObject('$in',[d['A']])) - def name = d['A'].replace('¾­¼Ã¼¼Êõ¿ª·¢Çø','¾­¼Ã¿ª·¢Çø') +// q.put("name", d['D']) + q.put("city",d['C']) +// q.put('alias', new BasicDBObject('$in',[d['D']])) + def name = d['D'] //.replace('¾­¼Ã¼¼Êõ¿ª·¢Çø','¾­¼Ã¿ª·¢Çø') q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //Ãû³Æ Ä£ºý²éѯ def obj = development_zone_CONN.findOne(q) if(obj){ - d['I'] = "¿âÖÐÓпª·¢Çø-ÏàËÆ2" + d['G'] = "ÏàËÆ" // d['K'] = obj.name // d['C'] = obj._id.toString() - d['B'] = obj.name -// if(obj.name == d['A']){ -// d['E'] = "Ïàͬ" + d['E'] = obj.name +// if(obj.name == d['D']){ +// d['G'] = "Ïàͬ" // }else{ -// d['E'] = "°üº¬" +// d['G'] = "°üº¬" // } + if(obj.shape){ + d['F'] = "ÊÇ" + } // d['E'] = "±ðÃû" // d['F'] = obj.name } } - },1); + },0); } def calAddrDistince(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevQlCheck.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevQlCheck.groovy new file mode 100644 index 000000000..f37e917d2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevQlCheck.groovy @@ -0,0 +1,113 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.ExcelUtil +import com.fangjia.script.gj.ProxyHttpUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.commons.io.FileUtils + +class DevQlCheck { + + //http://146.56.217.44:8088/uams/index + String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + static main(args) { + def task = new DevQlCheck() + task.run() + } + + def run(){ + def devName = "¸£½¨Áúº£¾­¼Ã¿ª·¢Çø" + def obj = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",devName)) + def pois = readCSV("C:\\Users\\gaojuan\\Downloads\\ÕÄÖÝÊÐPOI.csv") + def devPois = [] + devPois << ["Ê¡", "³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "µØÖ·", "ÀàÐÍ", "¾­¶È", "γ¶È", "ÆëÁº¿ª·¢Çø"] + for(def poi : pois){ + def flag = InsidePolygon.isInArea(poi.lng.toString().toDouble(), poi.lat.toString().toDouble(), InsidePolygon.createPoint4(obj.shape.coordinates)) + if(flag){ + devPois << [poi.province, poi.city, poi.district, poi.name, poi.address, poi.cat, poi.lng, poi.lat] + } + } + println "¹²£º"+devPois.size()+" ¸ö" + ExcelUtils.write("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\${devName}poiÆëÁºÐ£Ñé.xlsx", devPois, "ÆëÁº¿ª·¢Çø") + def index = 0 + ExcelUtils.modify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\${devName}poiÆëÁºÐ£Ñé.xlsx",{d-> + def address = d['B']+d['C']+d['D'] +// if(d['E']){ +// address = address + d['E'] +// } + Thread.sleep(3000) + def s = getQlResult(address) + d['I'] = s + index ++ + println index+" "+ s + }) + + + } + + def getQlResult(address){ + def params = [ + 'address': address, + 'type': '1' //1£º¶Ô¹« 0£º¶Ô˽ + ] + try{ + def str = ProxyHttpUtils.getProxyResult(targetUrl, params); + println address +" "+ str + JSONObject result = JSONObject.fromObject(str) + if(result && result.code==20000){ + def data = result.data + if(data){ + def list = data.indusParkList + if(list && list!="null"){ + def s = "" + for(def r : list){ + s = s + r.indusParkName +"-"+r.indusParkLevel+"\n" + } + return s + } + } + } + }catch(Exception e){ + println address +" "+e.getMessage() + } + + return null + } + + + def readCSV(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + for(line in lines) { + if(line){ + String[] arrs = line.split("\\|@\\|") + def o = new BasicDBObject() + try{ + o.put("province", arrs[0]) + o.put("city", arrs[1]) + o.put("district", arrs[2]) + o.put("name", arrs[3]) + o.put("township", arrs[4]) + o.put("address", arrs[5]) + o.put("cat", arrs[6]) + o.put("lng", arrs[7]) + o.put("lat", arrs[8]) + o.put("data_source", arrs[9]) + rets << o + }catch(Exception e){ + println filename+" ÎļþÒì³£" + } + + } + + } + return rets; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index aaf80284e..50ec412b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -32,7 +32,7 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() - def path = "D:\\Êý¾Ý\\¿ª·¢Çø-Ô°Çø.xlsx"//¿ª·¢Çø-Ô°Çø.xlsx + def path = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\¿ª·¢Çø-Ô°Çø.xlsx"//¿ª·¢Çø-Ô°Çø.xlsx // task.addDev(path) task.addDevAddress(path) // @@ -167,7 +167,7 @@ class DevUpdateTestBatch { println d['A'] +" ²»´æÔÚ" } } - },0) + },1) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index ee54817e6..8a5fc6eec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -1,12 +1,14 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.ccbevaluation.PgsqlCommUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils @@ -28,14 +30,24 @@ class GXDdistrictMatch { static main(args) { def task = new GXDdistrictMatch(); // task.run() - task.addOldMatch2() +// task.addOldMatch2() // task.runDb() +// task.hainanMatch() // task.updateDistStatus()//»Ö¸´name_cat²»Õý³£Êý¾Ý // task.saveToDB() //±£´æÐÂÔöÐ¡Çø // task.restoreDist()//»Ö¸´Ð¡Çøstatus=0 // task.runMatch() + +// task.checkDist() + + task.addPrice() + + } + + def addPrice(){ + addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd')) } def addOldMatch(){ @@ -94,6 +106,46 @@ class GXDdistrictMatch { } + def hainanMatch(){ + def fileName = "D:\\Êý¾Ý\\Ð¡Çø\\º£ÄÏδƥÅ䵨ַ-1.xlsx" + ExcelUtils.multiThreadingModify(fileName, { d -> + if(d['D'] && d['F'] ){// +// def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['I']) +// if(jsonObject){ +//// if(jsonObject.get("district") && jsonObject.get("district")!='null'){ +//// d['AO'] = jsonObject.get("district") +//// } +// if(jsonObject.get("province") && jsonObject.get("province")!="null"){ +// d['AP'] = jsonObject.get("province") +// } +// if(jsonObject.get("city") && jsonObject.get("city")!="null"){ +// d['AI'] = jsonObject.get("city") +// } +// } + def formatCity = RegionData.formatCity(d['F']) + def fullCityObj = fullCityMap.get(formatCity) + if(fullCityObj){ +// def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E']) + + def obj = findDatasByCityAndName(formatCity, d['D'].replaceAll('[¡¤\\(\\)£¨£©]|¡¤', '')) + if(obj){ + d['U'] = "Â¥ÅÌץȡ¿â" + d['V'] = obj.city + d['W'] = obj.region + d['X'] = obj.name + d['Y'] = obj.property_type + d['Z'] = obj.address + def loc = obj.location + if(loc){ + d['AA'] = loc.x + d['AB'] = loc.y + } + } + } + } + }) + } + def runDb(){ def fileName = "D:\\Êý¾Ý\\¹úÐÅ´ï\\¹úÐÅ´ï11³ÇδƥÅäÊý¾Ý.xlsx" ExcelUtils.modify(fileName, { d -> @@ -238,43 +290,41 @@ class GXDdistrictMatch { def saveToDB(){ // def fileName = "D:\\Êý¾Ý\\¹úÐÅ´ï\\¹úÐÅ´ï11³ÇδƥÅäÊý¾Ý.xlsx" - def fileName = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\Ð¡Çø\\¶«Ý¸Î´Æ¥Åä³É½»Êý¾Ý.xlsx" + def fileName = "D:\\Êý¾Ý\\Ð¡Çø\\º£ÄÏÐÂÔöÐ¡Çø.xlsx" ExcelUtils.modify(fileName, { d -> - if(d['A']){ + if(d['A'] && d['C']){ def udata = new BasicDBObject() udata.put("city", d['A']) udata.put("region", d['B']) def name = d['C'].replaceAll('[¡¤\\(\\)£¨£©]|¡¤', '').replace("½¨ÉèÖÐ", "") udata.put("name", name) - def alias = [] - if(d['G'] && name!=d['G']){ - alias << d['G'] - } +// def alias = [] +// if(d['G'] && name!=d['G']){ +// alias << d['G'] +// } // if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ // alias << d['AI'] // } - if(alias){ - udata.put("alias", alias) - } +// if(alias){ +// udata.put("alias", alias) +// } - def addr = AddressUtil.formatAddress(d['A'], d['D']) + def addr = AddressUtil.formatAddress(d['A'], d['E']) if(addr){ udata.put("address", addr) } - udata.put("status", "0") - udata.put("name_cat", "0") + udata.put("status", 0) + udata.put("name_cat", 0) udata.put("cat", "district") - udata.put("username", "¸ß¾ê") +// udata.put("username", "¸ß¾ê") def estateType2 = "ÆÕͨסլ" def type = "" -// if(d['AH']){ -// type = d['AH'] -// }else{ -// type = d['Y'] -// } + if(d['D']){ + type = d['D'] + } if(type){ if(type.contains("±ðÊû")){ estateType2 = "±ðÊû" @@ -282,26 +332,50 @@ class GXDdistrictMatch { estateType2 = "ÉÌס" } } - udata.put("estateType2", estateType2) + udata.put("estate_type2", estateType2) + if(d['F']){ Long lng = d['F']*10000000000 - udata.put("lng", lng) - } - if(d['E']){ - Long lat = d['E'] * 10000000000 + Long lat = d['G'] * 10000000000 udata.put("lat", lat) - } - - def s = FangjiaClientUtils.saveOrUpdateDistrict(udata) - def opt = "add" - if(s && s=='³É¹¦'){ - saveOptLog(udata.city, udata.region, udata.name, opt) + udata.put("lng", lng) + // ¸ßµÂת°Ù¶È + def baidu = BaiDuApiUtils.geoconv(d['F'].toString().toDouble(), d['G'].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + if(lng2){ + Long lng2_new = lng2 *10000000000L + Long lat2_new = lat2 *10000000000L + udata.put("lng2", lng2_new) + udata.put("lat2", lat2_new) } } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + def markMap = [:] + markMap.put("data_source", "»úÆ÷Èë¿â") + udata.put("mark_map",markMap) +// udata.put("is_new", 0) + DbUtils.seaweed.save(udata) + saveOptLog(udata.city, udata.region, udata.name, "add") +// def s = FangjiaClientUtils.saveOrUpdateDistrict(udata) + println name +// def opt = "add" +// if(s && s=='³É¹¦'){ +// saveOptLog(udata.city, udata.region, udata.name, opt) +// } + } }) } + def checkDist(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡Çø\\º£ÄÏδƥÅ䵨ַ-1.xlsx", { d -> + if(d['V'] && d['W'] && d['X']){ + matching(d['V'], d['W'], d, d['X'], null, null) + } + }) + } def match(city, region, d){ d = matching(city, region, d, d['C'].replaceAll('[¡¤\\(\\)£¨£©]|¡¤', ''), d['D'], "") @@ -391,7 +465,7 @@ class GXDdistrictMatch { if(rets && rets.size()>=1){ d = setRowValue(rets[0], d, "Ãû³ÆÔÚµØÖ·ÖÐ"+str) }else{ - if (d['D']) { + if (d['D'] && alias) { rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[¡¤\\(\\)£¨£©]|¡¤', ''), "district") if (rets) { if (rets.size() >= 1) { @@ -449,14 +523,14 @@ class GXDdistrictMatch { def setRowValue(dbo, d, remark){ if (dbo) { - d['V'] = dbo.city - d['W'] = dbo.region - d['X'] = dbo.name - d['Y'] = dbo.estate_type2 - d['Z'] = dbo.status - d['AA'] = dbo.name_cat - d['U'] = remark - d['AH'] = dbo.address +// d['V'] = dbo.city +// d['W'] = dbo.region +// d['X'] = dbo.name +// d['Y'] = dbo.estate_type2 + d['AK'] = dbo.status +// d['AA'] = dbo.name_cat + d['AJ'] = remark +// d['AH'] = dbo.address } return d; } @@ -510,6 +584,33 @@ class GXDdistrictMatch { return null; } + + def findDatasByCityAndName(def city, def name){ + def sql = "select * from community where city='${city}' and name='${name}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + rets << d + println d.property_type + }) + + if(rets){ + def obj = null + for(def r : rets){ + if(r.property_type && (r.property_type.contains("סլ") || r.property_type.contains("±ðÊû") || r.property_type.contains("¹«Ô¢"))){ + obj = r + break + } + } + if(obj){ + return obj; + }else{ + return rets[0]; + } + + } + return null; + } + def saveOptLog(city, region, name, type){ def obj = new BasicDBObject(); obj.put("city",city); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index a87c0b154..ea28d2c0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -467,7 +467,7 @@ class NewGaoDePoiBatch extends BaseBatch{ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) def map = new HashMap() ElasticUtils89.executeQuery("192.168.0.89", "map_poi_2024", queryBuilder, { String id, Map d-> - def district = "" + def district = "¼ÎÓø¹ØÊÐ" if(d.district){ district = d.district } @@ -477,11 +477,12 @@ class NewGaoDePoiBatch extends BaseBatch{ } def o = new BasicDBObject() o.put("city", d.city) - o.put("district", d.district) + o.put("district", district) o.put("name", d.name) o.put("township", d.township) o.put("location", d.location) regionMap.put(d.name.replace("-","").replace("¡¤",""), o) +// map.put(district, regionMap) map.put(district, regionMap) }) return map diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy index 21432c057..bf7a84c06 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy @@ -6,7 +6,7 @@ import com.fangjia.script.util.ExcelUtils class PoiNineSplitBatch{ - static main(args) { + public static void main(args) { def task = new PoiNineSplitBatch() task.run() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 4d3e00e32..41588517e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -3,7 +3,9 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import org.apache.commons.io.FileUtils import org.elasticsearch.action.bulk.BulkRequest @@ -12,6 +14,13 @@ import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * ÇøÓò±ê×¼»¯ * 1.Ð¡Çø¿âÏçÕò½ÖµÀ±ê×¼»¯£¬°üº¬Ð¡Çø£¬ÉÌÆÌ£¬Ð´×ÖÂ¥ @@ -22,6 +31,7 @@ class TownStandardBatch extends BaseBatch{ def fullCityMap = CommonDatautils.queryFullNameCity() def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def updateList = [] def addAliasList = [] @@ -38,7 +48,8 @@ class TownStandardBatch extends BaseBatch{ def TownStandard(){ // townStandardByCat("office") - townStandardByPoi() +// townStandardByPoi() + addTownByTownShape() } def townStandardByCat(def cat){ //"ÈýÃ÷","ÁúÑÒ","ÄÏÆ½","ÕÄÖÝ","¸£ÖÝ","ÏÃÃÅ","ȪÖÝ","ÆÎÌï","ÄþµÂ", @@ -69,11 +80,15 @@ class TownStandardBatch extends BaseBatch{ updateList << ["Êý¾ÝID", "³ÇÊÐ","ÇøÓò","ÏçÕò½ÖµÀ","Ãû³Æ","״̬", "ÐÞ¸Ä"] addAliasList << ["³ÇÊÐ","ÇøÓò","ÏçÕò½ÖµÀ", "ÐÂÔö±ðÃû"] - //Õã½­£¬½­ËÕ£¬°²»Õ - def citys = ['º¼ÖÝ','Äþ²¨','ÎÂÖÝ','¼ÎÐË','̨ÖÝ','ºþÖÝ','ÉÜÐË','½ð»ª','ÖÛɽ','áéÖÝ','ÀöË®']//CommonDatautils.findAllCity() + def runProvinces = ["¸£½¨Ê¡","Õã½­Ê¡","½­ËÕÊ¡","°²»ÕÊ¡","ɽ¶«Ê¡","¸ÊËàÊ¡"] + def citys = CommonDatautils.findAllCity() + def index = 0 for(def city : citys){ - + index++ def data = fullCityMap.get(city) +// if(runProvinces.contains(data.province)){ +// continue +// } def fullCityName = null if(data){ fullCityName = data.city @@ -95,11 +110,16 @@ class TownStandardBatch extends BaseBatch{ def rets = readPoiCsv(fileName) println city +" "+rets.size() if(set.size()>1){ - poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap) + poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data) }else{ // haveStreetProcess(city, set, cat) } } + if(updateList.size()>50000){ + ExcelUtils.write("D:\\Êý¾Ý\\ÏçÕò½ÖµÀ´ý²éÊý¾Ý-${index}.xlsx", updateList, "Êý¾Ý") + ExcelUtils.write("D:\\Êý¾Ý\\ÏçÕò½ÖµÀ´ý²éÊý¾Ý-ÐèÐÂÔö±ðÃû-${index}.xlsx", addAliasList, "Êý¾Ý") + updateList.clear() + } }else{ println fileName+" ÎļþûÕÒµ½" } @@ -107,8 +127,7 @@ class TownStandardBatch extends BaseBatch{ println "ûÕÒµ½ÏàÓ¦µÄÈ«³Æ£º"+city } } - ExcelUtils.write("D:\\Êý¾Ý\\ÏçÕò½ÖµÀ´ý²éÊý¾Ý.xlsx", updateList, "Êý¾Ý") - ExcelUtils.write("D:\\Êý¾Ý\\ÏçÕò½ÖµÀ´ý²éÊý¾Ý-ÐèÐÂÔö±ðÃû.xlsx", addAliasList, "Êý¾Ý") + ExcelUtils.write("D:\\Êý¾Ý\\ÏçÕò½ÖµÀ´ý²éÊý¾Ý-×îºó.xlsx", updateList, "Êý¾Ý") } def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ @@ -168,7 +187,7 @@ class TownStandardBatch extends BaseBatch{ } - def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){ + def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data){ def results = [] if(fullCityName){ for(def d : rets){ @@ -221,21 +240,21 @@ class TownStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["township":updateTown]) results << m -// if(results.size()%100==0){ -// updateByEs(results) -// results.clear() -// } - updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown] + if(results.size()%100==0){ + updateByEs(results) + results.clear() + } + updateList << [d.id, d.province, d.city, d.district, d.name,d.township, d.status, updateTown] // println d.city +" "+d.district+" "+d.name +" "+d.township+" ÏçÕòÐÞ¸ÄΪ£º"+updateTown }else{ def m = [:] m.put(d.id, ["township":""]) -// results << m -// if(results.size()%100==0){ -// updateByEs(results) -// results.clear() -// } - updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""] + results << m + if(results.size()%100==0){ + updateByEs(results) + results.clear() + } + updateList << [d.id, d.province, d.city, d.district, d.name, d.township, d.status, ""] println d.city +" "+d.district+" "+d.name +" "+d.township+" ״̬ÐÞ¸ÄΪ£º¿ÕÖµ" } } @@ -244,10 +263,10 @@ class TownStandardBatch extends BaseBatch{ } } -// if(results.size()>0){ -// updateByEs(results) -// results.clear() -// } + if(results.size()>0){ + updateByEs(results) + results.clear() + } println fullCityName +" ´¦ÀíÍê³É" } @@ -322,18 +341,194 @@ class TownStandardBatch extends BaseBatch{ if(line){ String[] arrs = line.split("\\|@\\|") def o = new BasicDBObject() - o.put("city", arrs[0]) - o.put("district", arrs[1]) - o.put("name", arrs[2]) - o.put("township", arrs[3]) - o.put("status", arrs[4]) - o.put("id", arrs[5]) - rets << o + try{ + o.put("city", arrs[0]) + o.put("district", arrs[1]) + o.put("name", arrs[2]) + o.put("township", arrs[3]) + o.put("status", arrs[4]) + o.put("id", arrs[5]) + rets << o + }catch(Exception e){ + println arrs + } + } } return rets; } + def readCSV2(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + def index = 0 + for(line in lines) { + if(line && index>=1){ + String[] arrs = line.split(",") + def o = new BasicDBObject() + try{ + o.put("id", arrs[1]) + o.put("province", arrs[2]) + o.put("city", arrs[3]) + o.put("district", arrs[4]) + o.put("lng", arrs[5]) + o.put("lat", arrs[6]) + o.put("data_source", arrs[7]) + rets << o + }catch(Exception e){ + println arrs + } + } + index++ + } + return rets; + } + + def addTownByTownShape(){ + def i = 0 + def headers = ["_id","province","city","district","lng","lat","data_source","ÏçÕò½ÖµÀ", "ÐÂÇøÓò", "·½Ê½"] + while(i<50){ + def fileName = "/home/goojia/gj/new_poi/input/no_township_${i}.csv" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\ÏçÕò½ÖµÀ\\´ý²¹³äÏçÕò½ÖµÀ\\ԭʼ\\no_township_${i}.csv" + } + File file=new File(fileName); + if(file.exists()) { + def rets = readCSV2(fileName) + def rows = addTownshipBatch(rets) + if (CommonDatautils.isWindow()) { + com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\ÏçÕò½ÖµÀ\\´ý²¹³äÏçÕò½ÖµÀ\\½á¹û\\no_township_result_${i}.csv", headers, "utf-8", false, ',', ''); + }else{ + com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', ''); + } + println fileName +" ÔËÐÐÍê³É" + }else{ + println fileName +" ²»´æÔÚ" + } + i++ + } + } + + def addTownshipBatch(def rets){ + def results = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i= 1) { + addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()]) + results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source] + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + return results + } + + def addTown(saveObj, townMap, location) { + //ÔÚpoi±¾ÇøÓòÏÂͨ¹ýÏçÕò½ÖµÀΧÀ¸Ñ¡³öËùÔÚÏçÕò½ÖµÀ + def towns = townMap.get(saveObj.district) + if (towns) { + for (def town : towns) { + if (town.shape && town.shape.coordinates) { + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if (flag) { + saveObj.township = town.street + saveObj.town_source = "Í¬ÇøÓò" + break + } + } + } + } + //ºöÂÔpoiÖеÄÇøÓò ͨ¹ýÏçÕò½ÖµÀΧÀ¸Ñ¡³öËùÔÚÏçÕò½ÖµÀ + if(!saveObj.township){ + def townNames = townMap.keySet() + for(def townName : townNames){ + towns = townMap.get(townName) + for (def town : towns) { + if (town.shape && town.shape.coordinates) { + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if (flag) { + saveObj.township = town.street + saveObj.new_district = town.mzb_district_name + saveObj.town_source = "ºöÂÔÇøÓò" + break + } + } + } + if(saveObj.township){ + break; + } + } + } + //ͨ¹ýµ÷È¡µØÍ¼api½Ó¿Ú»ñÈ¡ÏçÕò½ÖµÀ +// if(!saveObj.township){ +// def gaodeMap = getTownshipInfo(location.lon+","+location.lat) +// if(gaodeMap && gaodeMap.township){ +// saveObj.township = gaodeMap.township +// saveObj.new_district = gaodeMap.district +// saveObj.town_source = "¸ßµÂAPI" +// } +// } + return saveObj + } + + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + city = addressComponent.city + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index d99e249c9..bbe074507 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -38,16 +38,16 @@ class TownshipStrandUpdate { static main(args) { def task = new TownshipStrandUpdate() -// task.process() -// task.updateTown() + task.process() + task.updateTown() // task.stat() - task.updateDiffRegionTown() +// task.updateDiffRegionTown() // task.cleanTown() // task.checkTownFromMongo() // task.updateTownMapping() // task.updateDevMapping() - println new Date().getTime() +// println new Date().getTime() // task.updateSpecCityTown() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 3b3aed61a..d6e73b168 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{ ] static main(args) { - runBatch(new DistrictFieldAddBatch(), args) +// runBatch(new DistrictFieldAddBatch(), args) + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡Çø\\ÉϺ£±ðÊû.xlsx",{d-> + def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district") +// if(dist){ +// d['L'] = dist.loudong_count +// d['M'] = dist.household_count +// d['Q'] = dist.estate_type2 +// } + def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "ËÑ·¿") + if(!list && dist){ + def alias = dist.alias + for(def name : alias){ + list = findEsDatasByCityAndName(d['B'], d['G'], name, "ËÑ·¿") + if(list){ + d['Z'] = "±ðÃû" + } + } + }else{ + d['Z'] = "Ãû³Æ" + } + if(list){ + def obj = list[0] + if(obj.loudong_cnt){ + d['W'] = obj.loudong_cnt + } + + if(obj.households){ + d['X'] = obj.households + } + d['Y'] = obj.url + } + + }) } @Override @@ -361,6 +393,15 @@ class DistrictFieldAddBatch extends BaseBatch{ return list } + + static def findEsDatasByCityAndName(c, district, name, esWebSource){ + def list = [] + ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' and region='${district}' and name='${name}'", {r-> + list << r + }) + return list + } + def fieldUpdate(total,updateRets){ def countMap = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 15e4cc80b..35179b3d9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -398,4 +398,32 @@ class FangjiaClientUtils { return jsonObj; } + + /** + * ¾­Î³¶Èת»» + * 1¡¢¸ßµÂת°Ù¶È 2¡¢¸ßµÂתWGS84 3¡¢°Ù¶Èת¸ßµÂ 4¡¢WGS84ת¸ßµÂ 5¡¢°Ù¶ÈתWGS84 6¡¢WGS84ת°Ù¶È + */ + public static def convert(String locationstr, type){ + def jsonObj = null + def content = null + def points = [] + try { + def url = "http://192.168.0.19:31080/FangJiaNlpService/convert?locations="+URLEncoder.encode(locationstr,"UTF-8")+"&coordsys=${type}" + + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + if(jsonObj && jsonObj.locations){ + def locations = jsonObj.locations + for(def s : locations){ + def arrs = s.split(",") + points = [arrs[0].toString().toDouble(), arrs[1].toString().toDouble()] + } + } + return points + } catch (Exception e) { + println e.getMessage() + } + return null; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy index 99b913307..3ea63b7e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -13,18 +13,18 @@ class MsbPocTest { task.run(); } def run(){ - def inputfile = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\_ÒµÎñ5Ô·Ýϵͳ·´À¡ÎÊÌâ¼Û¸ñºË²é20240416-20240516 - ¹úÐÅ´ï0524.xlsx" + def inputfile = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\·¿Ñ¶Í¨10Ô¼۸ñºË²é_1126-FXTÌá½»½á¹û20241205.xlsx" // def inputfile = "D:\\Êý¾Ý\\ÃÀÊõ±¦\\ÃÀÊõ±¦²âÊÔÊý¾Ý(Æ¥ÅäV2).xlsx"//ÃÀÊõ±¦²âÊÔÊý¾Ý(Æ¥Åä).xlsx // standard(inputfile) // match(inputfile) -// matchOld(inputfile) + matchOld(inputfile) // addPrice("D:\\Êý¾Ý\\ÃÀÊõ±¦\\ÃÀÊõ±¦²âÊÔÊý¾Ý(¼Û¸ñ).xlsx") // test("D:\\Êý¾Ý\\ÃÀÊõ±¦\\test2.xlsx") // cityPrice() // mergeMatchPrice("D:\\Êý¾Ý\\ÃÀÊõ±¦\\ÃÀÊõ±¦-нӿڼ۸ñ.xlsx") - addCCBPrice(inputfile) +// addCCBPrice(inputfile) } def cityPrice(){ @@ -69,24 +69,24 @@ class MsbPocTest { def matchOld(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(!row['Y'] && row['G']){ + if(!row['AD'] ){ def city = row['C'] def region = null //row['D'] - def name = row['E'] - def address = row['G'] + def name = null// row['E'] + def address = row['D'] println "[match] city:"+city+" region:"+region+" address:"+address def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) if(jsonObject != null && jsonObject.precise!=-1) { - row['Y'] = jsonObject.get("id") - row['Z'] = jsonObject.get("city") - row['AA'] = jsonObject.get("district") - row['AB'] = jsonObject.get("name") - row['AC'] = jsonObject.get("propertyType") + row['AD'] = jsonObject.get("id") + row['AE'] = jsonObject.get("city") + row['AF'] = jsonObject.get("district") + row['AG'] = jsonObject.get("name") + row['AH'] = jsonObject.get("propertyType") // row['X'] = jsonObject.get("credit") } } - }) + },1) } @@ -114,7 +114,7 @@ class MsbPocTest { if(!row['Y']){ def city = row['C'] def region = null //row['B'] - def address = row['G'] //row['G']//row['F'] + def address = row['D'] //row['G']//row['F'] println "[match] city:"+city+" region:"+region+" address:"+address def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false) @@ -125,7 +125,7 @@ class MsbPocTest { row['AB'] = jsonObject.get("name") } } - }) + },1) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index d314427a9..5cd666999 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -38,9 +38,9 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\202410²âÊÔÆ«²î½Ï´ó°¸Àýͳ¼ÆÊý¾Ýtest.xlsx" - addGXDPrice(filePath) -// addCcbPrice(filePath) + def filePath = "D:\\Êý¾Ý\\¼Û¸ñ\\»ÝÖÝ.xlsx" +// addGXDPrice(filePath) + addCcbPrice(filePath) // addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() @@ -174,26 +174,28 @@ class AddPriceTest { def addCcbPrice(filepath){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(filepath,{ d-> - if(d['U']){ - def cityId = cityCodeDataMap.get(d['V']) - def map = findAvgPrice(d['U'], cityId) + if(d['A']){ + def cityId = cityCodeDataMap.get(d['B']) + def map = findAvgPrice(d['A'], cityId) if(map){ - d['H'] = map.avg_price - d['I'] = map.calc_type + d['I'] = map.avg_price + d['J'] = map.month +// d['H'] = map.calc_type // d['AM'] = map.credit_value } -// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId) -// if(newPriceMap){ -// d['Q'] = newPriceMap.avg_price -// } + def newPriceMap = findNewFjwAvgPrice(d['A'], cityId) + if(newPriceMap){ + d['G'] = newPriceMap.avg_price + d['H'] = newPriceMap.month + } } }) } def findAvgPrice(def commid, def cityid){ - def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1" + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" def map = [:] pgsqlInstance.eachRow(sql, { r -> map.avg_price = r.gujia_bds_clc_avprc @@ -213,7 +215,7 @@ class AddPriceTest { def findNewFjwAvgPrice(def commid, def cityid){ - def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1" + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" def map = [:] pgsqlInstance235.eachRow(sql, { r -> map.avg_price = r.gujia_bds_clc_avprc @@ -270,14 +272,14 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['B']) { + if (!d['H']) { def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) if(gxdPrice){ - d['B'] = gxdPrice.price - d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd") + d['H'] = gxdPrice.price + d['I'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd") } } @@ -305,11 +307,11 @@ class AddPriceTest { ExcelUtils.multiThreadingModify(path,{d-> // if(!d['L']){ // if(d['J'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['AJ']) + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ // d['A'] = dist.city // d['B'] = dist.region - d['E'] = dist.name +// d['E'] = dist.name // d['D'] = dist.status // d['E'] = dist.name_cat // d['F'] = dist.estate_type2 @@ -318,9 +320,9 @@ class AddPriceTest { // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ - d['G'] = dbo.avg_price + d['E'] = dbo.avg_price_2b // d['X'] = dbo.cal_type -// d['E'] = dbo.avg_price_2b +// d['E'] = dbo.avg_price } } // } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/GeoHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/GeoHelper.groovy index cb59203cf..e04562271 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/GeoHelper.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/GeoHelper.groovy @@ -1,4 +1,8 @@ package com.fangjia.script.util + + +import java.text.DecimalFormat + class GeoHelper { @@ -60,10 +64,159 @@ class GeoHelper { // return GetLatLon(a, distance, angle); // } // - public static void main(String[] args){ - GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0); - GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462); - } + public static haversine(lat1, lon1, lat2, lon2) { + def R = 6371; // µØÇò°ë¾¶£¨¹«À + def lat1Rad = lat1 * (Math.PI / 180); + def lat2Rad = lat2 * (Math.PI / 180); + def deltaLat = (lat2 - lat1) * (Math.PI / 180); + def deltaLon = (lon2 - lon1) * (Math.PI / 180); + + def a = Math.sin(deltaLat / 2) ** 2 + + Math.cos(lat1Rad) * Math.cos(lat2Rad) * + Math.sin(deltaLon / 2) ** 2; + + def c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); + return R * c; // ·µ»Ø¾àÀ루¹«À + } + + public static calculateArea(coords) { + def area = 0; + def n = coords.size(); + + for (def i = 0; i < n; i++) { + def j = (i + 1) % n; + area += coords[i][0] * coords[j][1]; + area -= coords[j][0] * coords[i][1]; + } + area = Math.abs(area) / 2; // ƽ·½¶È + + // ת»»ÎªÆ½·½¹«Àï + def conversionFactor = 111.32; // ´óԼÿ¶È¾­Î³¶ÈµÄ¹«ÀïÊý£¨¿¼Âǵ½µØÇòµÄÇúÂÊ£© + + def df = new DecimalFormat("#.000000") + return Double.parseDouble(df.format(area * conversionFactor * conversionFactor)) // ·µ»ØÃæ»ý£¨Æ½·½¹«À +// return area * conversionFactor * conversionFactor + } + + + public static void main(String[] args){ +// GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0); +// GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462); + + List> coordinates = [ + [ + 117.61878540582734, + 24.667331307258223 + ], + [ + 117.61400840589488, + 24.670008244935104 + ], + [ + 117.60936140565889, + 24.671837202755682 + ], + [ + 117.60834440585768, + 24.668667276729821 + ], + [ + 117.60696440583342, + 24.667256309201207 + ], + [ + 117.60517040553236, + 24.667762297657362 + ], + [ + 117.60272340547806, + 24.660100475272998 + ], + [ + 117.60254340555321, + 24.658630509428463 + ], + [ + 117.60206940590035, + 24.654781598282568 + ], + [ + 117.60254840557604, + 24.652731645907636 + ], + [ + 117.60346440599301, + 24.651555673447035 + ], + [ + 117.6070314057805, + 24.652578649754957 + ], + [ + 117.61605140571805, + 24.651939664372833 + ], + [ + 117.6156454058372, + 24.654165613124405 + ], + [ + 117.61559340595863, + 24.655023593107103 + ], + [ + 117.61535440558418, + 24.658990501065318 + ], + [ + 117.61662340528017, + 24.659352492642654 + ], + [ + 117.61678740531192, + 24.661365445721003 + ], + [ + 117.61472940542001, + 24.661623439403797 + ], + [ + 117.61462940586114, + 24.661837434759093 + ], + [ + 117.61470140583106, + 24.662665415879701 + ], + [ + 117.61487240553515, + 24.663165403873506 + ], + [ + 117.61522940591078, + 24.663494396452791 + ], + [ + 117.6157014059138, + 24.663523395394918 + ], + [ + 117.61627240565099, + 24.663438397829282 + ], + [ + 117.61693540544979, + 24.663181403867725 + ], + [ + 117.61878540582734, + 24.667331307258223 + ] + ] + + double area = calculateArea(coordinates) + println "The area of the polygon is: ${String.format("%.2f", area)} square kilometers" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 65da1a7be..7c41bcc1c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //±¦½£ + def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //±¦½£:c6fcd4bfbc40d73ee762b8115dca3390 //15021384995£ºdfc2245bac8a7b2d5b62e060e59d4eca /**