This commit is contained in:
2023-11-22 10:27:01 +08:00
parent c9355a86a1
commit 5d8e2bb943
6 changed files with 106 additions and 28 deletions
@@ -3,6 +3,7 @@ package com.fangjia.script.commData
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.JacksonUtil
@@ -90,6 +91,9 @@ class CommonDatautils {
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com") public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>() private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{ static{
@@ -4217,6 +4221,17 @@ class CommonDatautils {
return price return price
} }
static def findTownRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
} }
@@ -23,7 +23,7 @@ class SuzouPocTest {
} }
def run(){ def run(){
def path = "D:\\数据\\江苏农行\\江苏农1062条.xlsx" def path = "D:\\数据\\江苏农行\\江苏农1062条(合).xlsx"
addressEvalute(path) addressEvalute(path)
// addFjwTobPrice(path) // addFjwTobPrice(path)
// addThirdPrice(path) // addThirdPrice(path)
@@ -33,7 +33,7 @@ class SuzouPocTest {
def addressEvalute(path){ def addressEvalute(path){
ExcelUtils.modify(path,{d-> ExcelUtils.multiThreadingModify(path,{d->
if(!d['O'] && d['H']){ if(!d['O'] && d['H']){
def city = d['D'] def city = d['D']
@@ -8,6 +8,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
@@ -41,21 +42,37 @@ class ApiTest {
} }
def addrStructure(path){ def addrStructure(path){
ExcelUtils.modify(path,{d-> ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){ if(d['B']){
def jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']); JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject && jsonObject.containsKey("result")) { if(jsonObject) {
d['C'] = jsonObject.get("province") if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['D'] = jsonObject.get("city") d['C'] = jsonObject.get("province")
d['E'] = jsonObject.get("district") }
d['F'] = jsonObject.get("township") if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['G'] = jsonObject.get("address") d['D'] = jsonObject.get("city")
d['H'] = jsonObject.get("provinceCode") }
d['I'] = jsonObject.get("cityCode") if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['J'] = jsonObject.get("districtCode") d['E'] = jsonObject.get("district")
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
d['F'] = jsonObject.get("township")
}
if(jsonObject.get("address") && jsonObject.get("address")!="null"){
d['G'] = jsonObject.get("address")
}
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
d['H'] = jsonObject.get("provinceCode")
}
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
d['I'] = jsonObject.get("cityCode")
}
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
d['J'] = jsonObject.get("districtCode")
}
} }
} }
},0); },4);
} }
def addressEvaluate(path){ def addressEvaluate(path){
@@ -319,6 +319,7 @@ class FangjiaClientUtils {
} }
content = new HttpRequest().newGet(url, null, "utf-8") content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content) jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) { } catch (Exception e) {
println address +" "+content println address +" "+content
e.printStackTrace(); e.printStackTrace();
@@ -4,9 +4,11 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData import com.fangjia.script.util.ProvinceData
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
@@ -44,36 +46,36 @@ class SuggestionDataSynch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
// run() run()
updateCode() // updateCode()
} }
def run() { def run() {
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆" //"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
def citys = CommonDatautils.findAllCity() def citys = CommonDatautils.findAllCity().reverse()
def index = 0 def index = 0
fullCityMap = queryFullNameCity() fullCityMap = queryFullNameCity()
mcaDistMap = queryDistNameCity() mcaDistMap = queryDistNameCity()
for (def c : citys) { for (def c : citys) {
index++ index++
if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){ // if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
updating(c, "district") updating(c, "district")
updating(c, "office") updating(c, "office")
updating(c, "shop") updating(c, "shop")
println index + " " + c + " 完成" println index + " " + c + " 完成"
// }
if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
poiData(c)
} }
// poiData(c)
} }
} }
def updating(city, cat){ def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"]) def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
def rets = new CopyOnWriteArrayList() def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
@@ -143,6 +145,7 @@ class SuggestionDataSynch extends BaseBatch{
map.location = loc map.location = loc
} }
map.source = "FJW" map.source = "FJW"
map.township = dist.township
return map return map
} }
@@ -171,17 +174,18 @@ class SuggestionDataSynch extends BaseBatch{
def poiData(city){ def poiData(city){
def townMap = CommonDatautils.findTownRange(city)
def data = fullCityMap.get(city) def data = fullCityMap.get(city)
if(data){ if(data){
def results = CommonDatautils.findAmapDetailBySql(data.city) def results = CommonDatautils.findAmapDetailBySql(data.city)
createPoi(results, data) createPoi(results, data, townMap)
println city +" 完成!" println city +" 完成!"
} }
} }
def createPoi(results, data){ def createPoi(results, data, townMap){
def rets = new CopyOnWriteArrayList() def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
@@ -208,6 +212,15 @@ class SuggestionDataSynch extends BaseBatch{
map.cat = dist.cat map.cat = dist.cat
map.location = dist.location map.location = dist.location
map.source = "FJW" map.source = "FJW"
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
}
println map.township
//往100 上同步 //往100 上同步
updateData(map) updateData(map)
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
@@ -242,9 +255,9 @@ class SuggestionDataSynch extends BaseBatch{
} }
} }
} }
}else{ }else if(d.level == "区域"){
if(d.city == "省直辖县级行政区划" && d.district){ if(d.administer == "省直辖县级行政区划"){
fjwcity = RegionData.formatCity(d.district) fjwcity = RegionData.formatCity(d.city)
if(fjwcity){ if(fjwcity){
d.city = d.district d.city = d.district
} }
@@ -305,4 +318,36 @@ class SuggestionDataSynch extends BaseBatch{
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)); administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
} }
} }
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
} }
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size()) def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 5 def nThread = Runtime.getRuntime().availableProcessors() * 3
println nThread println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread) ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0) final AtomicInteger counter = new AtomicInteger(0)