This commit is contained in:
2023-11-22 10:27:01 +08:00
parent c9355a86a1
commit 5d8e2bb943
6 changed files with 106 additions and 28 deletions
@@ -3,6 +3,7 @@ package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
@@ -90,6 +91,9 @@ class CommonDatautils {
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
@@ -4217,6 +4221,17 @@ class CommonDatautils {
return price
}
static def findTownRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
}
@@ -23,7 +23,7 @@ class SuzouPocTest {
}
def run(){
def path = "D:\\数据\\江苏农行\\江苏农1062条.xlsx"
def path = "D:\\数据\\江苏农行\\江苏农1062条(合).xlsx"
addressEvalute(path)
// addFjwTobPrice(path)
// addThirdPrice(path)
@@ -33,7 +33,7 @@ class SuzouPocTest {
def addressEvalute(path){
ExcelUtils.modify(path,{d->
ExcelUtils.multiThreadingModify(path,{d->
if(!d['O'] && d['H']){
def city = d['D']
@@ -8,6 +8,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -41,21 +42,37 @@ class ApiTest {
}
def addrStructure(path){
ExcelUtils.modify(path,{d->
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject && jsonObject.containsKey("result")) {
d['C'] = jsonObject.get("province")
d['D'] = jsonObject.get("city")
d['E'] = jsonObject.get("district")
d['F'] = jsonObject.get("township")
d['G'] = jsonObject.get("address")
d['H'] = jsonObject.get("provinceCode")
d['I'] = jsonObject.get("cityCode")
d['J'] = jsonObject.get("districtCode")
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject) {
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['C'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['D'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['E'] = jsonObject.get("district")
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
d['F'] = jsonObject.get("township")
}
if(jsonObject.get("address") && jsonObject.get("address")!="null"){
d['G'] = jsonObject.get("address")
}
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
d['H'] = jsonObject.get("provinceCode")
}
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
d['I'] = jsonObject.get("cityCode")
}
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
d['J'] = jsonObject.get("districtCode")
}
}
}
},0);
},4);
}
def addressEvaluate(path){
@@ -319,6 +319,7 @@ class FangjiaClientUtils {
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
@@ -4,9 +4,11 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@@ -44,36 +46,36 @@ class SuggestionDataSynch extends BaseBatch{
@Override
public void process(Object args) {
// run()
updateCode()
run()
// updateCode()
}
def run() {
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
def citys = CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity().reverse()
def index = 0
fullCityMap = queryFullNameCity()
mcaDistMap = queryDistNameCity()
for (def c : citys) {
index++
if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
updating(c, "district")
updating(c, "office")
updating(c, "shop")
println index + " " + c + " 完成"
// }
if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
poiData(c)
}
// poiData(c)
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"])
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
@@ -143,6 +145,7 @@ class SuggestionDataSynch extends BaseBatch{
map.location = loc
}
map.source = "FJW"
map.township = dist.township
return map
}
@@ -171,17 +174,18 @@ class SuggestionDataSynch extends BaseBatch{
def poiData(city){
def townMap = CommonDatautils.findTownRange(city)
def data = fullCityMap.get(city)
if(data){
def results = CommonDatautils.findAmapDetailBySql(data.city)
createPoi(results, data)
createPoi(results, data, townMap)
println city +" 完成!"
}
}
def createPoi(results, data){
def createPoi(results, data, townMap){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
@@ -208,6 +212,15 @@ class SuggestionDataSynch extends BaseBatch{
map.cat = dist.cat
map.location = dist.location
map.source = "FJW"
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
}
println map.township
//往100 上同步
updateData(map)
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
@@ -242,9 +255,9 @@ class SuggestionDataSynch extends BaseBatch{
}
}
}
}else{
if(d.city == "省直辖县级行政区划" && d.district){
fjwcity = RegionData.formatCity(d.district)
}else if(d.level == "区域"){
if(d.administer == "省直辖县级行政区划"){
fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
d.city = d.district
}
@@ -305,4 +318,36 @@ class SuggestionDataSynch extends BaseBatch{
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
}
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
}
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 5
def nThread = Runtime.getRuntime().availableProcessors() * 3
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)