update
This commit is contained in:
+15
@@ -3,6 +3,7 @@ package com.fangjia.script.commData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
@@ -90,6 +91,9 @@ class CommonDatautils {
|
||||
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
@@ -4217,6 +4221,17 @@ class CommonDatautils {
|
||||
return price
|
||||
}
|
||||
|
||||
static def findTownRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -23,7 +23,7 @@ class SuzouPocTest {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def path = "D:\\数据\\江苏农行\\江苏农1062条.xlsx"
|
||||
def path = "D:\\数据\\江苏农行\\江苏农1062条(合).xlsx"
|
||||
addressEvalute(path)
|
||||
// addFjwTobPrice(path)
|
||||
// addThirdPrice(path)
|
||||
@@ -33,7 +33,7 @@ class SuzouPocTest {
|
||||
|
||||
def addressEvalute(path){
|
||||
|
||||
ExcelUtils.modify(path,{d->
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
|
||||
if(!d['O'] && d['H']){
|
||||
def city = d['D']
|
||||
|
||||
+29
-12
@@ -8,6 +8,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -41,21 +42,37 @@ class ApiTest {
|
||||
}
|
||||
|
||||
def addrStructure(path){
|
||||
ExcelUtils.modify(path,{d->
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['B']){
|
||||
def jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
d['C'] = jsonObject.get("province")
|
||||
d['D'] = jsonObject.get("city")
|
||||
d['E'] = jsonObject.get("district")
|
||||
d['F'] = jsonObject.get("township")
|
||||
d['G'] = jsonObject.get("address")
|
||||
d['H'] = jsonObject.get("provinceCode")
|
||||
d['I'] = jsonObject.get("cityCode")
|
||||
d['J'] = jsonObject.get("districtCode")
|
||||
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
|
||||
if(jsonObject) {
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['C'] = jsonObject.get("province")
|
||||
}
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['D'] = jsonObject.get("city")
|
||||
}
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['E'] = jsonObject.get("district")
|
||||
}
|
||||
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
|
||||
d['F'] = jsonObject.get("township")
|
||||
}
|
||||
if(jsonObject.get("address") && jsonObject.get("address")!="null"){
|
||||
d['G'] = jsonObject.get("address")
|
||||
}
|
||||
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
|
||||
d['H'] = jsonObject.get("provinceCode")
|
||||
}
|
||||
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
|
||||
d['I'] = jsonObject.get("cityCode")
|
||||
}
|
||||
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
|
||||
d['J'] = jsonObject.get("districtCode")
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
},4);
|
||||
}
|
||||
|
||||
def addressEvaluate(path){
|
||||
|
||||
+1
@@ -319,6 +319,7 @@ class FangjiaClientUtils {
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
|
||||
+58
-13
@@ -4,9 +4,11 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
@@ -44,36 +46,36 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// run()
|
||||
updateCode()
|
||||
run()
|
||||
// updateCode()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run() {
|
||||
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def index = 0
|
||||
fullCityMap = queryFullNameCity()
|
||||
mcaDistMap = queryDistNameCity()
|
||||
|
||||
for (def c : citys) {
|
||||
index++
|
||||
if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
||||
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
||||
updating(c, "district")
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
println index + " " + c + " 完成"
|
||||
|
||||
// }
|
||||
if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
|
||||
poiData(c)
|
||||
}
|
||||
|
||||
// poiData(c)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"])
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
@@ -143,6 +145,7 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
map.location = loc
|
||||
}
|
||||
map.source = "FJW"
|
||||
map.township = dist.township
|
||||
return map
|
||||
}
|
||||
|
||||
@@ -171,17 +174,18 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
|
||||
def poiData(city){
|
||||
def townMap = CommonDatautils.findTownRange(city)
|
||||
def data = fullCityMap.get(city)
|
||||
if(data){
|
||||
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
||||
createPoi(results, data)
|
||||
createPoi(results, data, townMap)
|
||||
println city +" 完成!"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def createPoi(results, data){
|
||||
def createPoi(results, data, townMap){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
@@ -208,6 +212,15 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
map.cat = dist.cat
|
||||
map.location = dist.location
|
||||
map.source = "FJW"
|
||||
|
||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||
if(!map.township){
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
}
|
||||
println map.township
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
||||
@@ -242,9 +255,9 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(d.city == "省直辖县级行政区划" && d.district){
|
||||
fjwcity = RegionData.formatCity(d.district)
|
||||
}else if(d.level == "区域"){
|
||||
if(d.administer == "省直辖县级行政区划"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
d.city = d.district
|
||||
}
|
||||
@@ -305,4 +318,36 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
|
||||
}
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
}
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 3
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
Reference in New Issue
Block a user