update
This commit is contained in:
+15
@@ -3,6 +3,7 @@ package com.fangjia.script.commData
|
|||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
import com.fangjia.httpclient.UserAgents
|
import com.fangjia.httpclient.UserAgents
|
||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
import com.fangjia.script.util.DbUtils
|
import com.fangjia.script.util.DbUtils
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.JacksonUtil
|
import com.fangjia.script.util.JacksonUtil
|
||||||
@@ -90,6 +91,9 @@ class CommonDatautils {
|
|||||||
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
|
||||||
|
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||||
|
|
||||||
|
|
||||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||||
static{
|
static{
|
||||||
@@ -4217,6 +4221,17 @@ class CommonDatautils {
|
|||||||
return price
|
return price
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def findTownRange(city){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city",city)
|
||||||
|
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||||
|
def townMap = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||||
|
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||||
|
}
|
||||||
|
return townMap
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -23,7 +23,7 @@ class SuzouPocTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def path = "D:\\数据\\江苏农行\\江苏农1062条.xlsx"
|
def path = "D:\\数据\\江苏农行\\江苏农1062条(合).xlsx"
|
||||||
addressEvalute(path)
|
addressEvalute(path)
|
||||||
// addFjwTobPrice(path)
|
// addFjwTobPrice(path)
|
||||||
// addThirdPrice(path)
|
// addThirdPrice(path)
|
||||||
@@ -33,7 +33,7 @@ class SuzouPocTest {
|
|||||||
|
|
||||||
def addressEvalute(path){
|
def addressEvalute(path){
|
||||||
|
|
||||||
ExcelUtils.modify(path,{d->
|
ExcelUtils.multiThreadingModify(path,{d->
|
||||||
|
|
||||||
if(!d['O'] && d['H']){
|
if(!d['O'] && d['H']){
|
||||||
def city = d['D']
|
def city = d['D']
|
||||||
|
|||||||
+29
-12
@@ -8,6 +8,7 @@ import com.fangjia.script.util.MongoUtils
|
|||||||
import com.fangjia.script.util.OpenApiClient
|
import com.fangjia.script.util.OpenApiClient
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
@@ -41,21 +42,37 @@ class ApiTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def addrStructure(path){
|
def addrStructure(path){
|
||||||
ExcelUtils.modify(path,{d->
|
ExcelUtils.multiThreadingModify(path,{d->
|
||||||
if(d['B']){
|
if(d['B']){
|
||||||
def jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
|
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
|
||||||
if(jsonObject && jsonObject.containsKey("result")) {
|
if(jsonObject) {
|
||||||
d['C'] = jsonObject.get("province")
|
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||||
d['D'] = jsonObject.get("city")
|
d['C'] = jsonObject.get("province")
|
||||||
d['E'] = jsonObject.get("district")
|
}
|
||||||
d['F'] = jsonObject.get("township")
|
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||||
d['G'] = jsonObject.get("address")
|
d['D'] = jsonObject.get("city")
|
||||||
d['H'] = jsonObject.get("provinceCode")
|
}
|
||||||
d['I'] = jsonObject.get("cityCode")
|
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||||
d['J'] = jsonObject.get("districtCode")
|
d['E'] = jsonObject.get("district")
|
||||||
|
}
|
||||||
|
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
|
||||||
|
d['F'] = jsonObject.get("township")
|
||||||
|
}
|
||||||
|
if(jsonObject.get("address") && jsonObject.get("address")!="null"){
|
||||||
|
d['G'] = jsonObject.get("address")
|
||||||
|
}
|
||||||
|
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
|
||||||
|
d['H'] = jsonObject.get("provinceCode")
|
||||||
|
}
|
||||||
|
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
|
||||||
|
d['I'] = jsonObject.get("cityCode")
|
||||||
|
}
|
||||||
|
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
|
||||||
|
d['J'] = jsonObject.get("districtCode")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},0);
|
},4);
|
||||||
}
|
}
|
||||||
|
|
||||||
def addressEvaluate(path){
|
def addressEvaluate(path){
|
||||||
|
|||||||
+1
@@ -319,6 +319,7 @@ class FangjiaClientUtils {
|
|||||||
}
|
}
|
||||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||||
jsonObj = JSONObject.fromObject(content)
|
jsonObj = JSONObject.fromObject(content)
|
||||||
|
println jsonObj
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
println address +" "+content
|
println address +" "+content
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
|||||||
+58
-13
@@ -4,9 +4,11 @@ import com.fangjia.core.RegionData
|
|||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.fangjia.script.util.ProvinceData
|
import com.fangjia.script.util.ProvinceData
|
||||||
|
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
@@ -44,36 +46,36 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
// run()
|
run()
|
||||||
updateCode()
|
// updateCode()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def run() {
|
def run() {
|
||||||
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
|
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
|
||||||
def citys = CommonDatautils.findAllCity()
|
def citys = CommonDatautils.findAllCity().reverse()
|
||||||
def index = 0
|
def index = 0
|
||||||
fullCityMap = queryFullNameCity()
|
fullCityMap = queryFullNameCity()
|
||||||
mcaDistMap = queryDistNameCity()
|
mcaDistMap = queryDistNameCity()
|
||||||
|
|
||||||
for (def c : citys) {
|
for (def c : citys) {
|
||||||
index++
|
index++
|
||||||
if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
||||||
updating(c, "district")
|
updating(c, "district")
|
||||||
updating(c, "office")
|
updating(c, "office")
|
||||||
updating(c, "shop")
|
updating(c, "shop")
|
||||||
println index + " " + c + " 完成"
|
println index + " " + c + " 完成"
|
||||||
|
// }
|
||||||
|
if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
|
||||||
|
poiData(c)
|
||||||
}
|
}
|
||||||
|
|
||||||
// poiData(c)
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def updating(city, cat){
|
def updating(city, cat){
|
||||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"])
|
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
|
||||||
def rets = new CopyOnWriteArrayList()
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
|
||||||
def totalCount = new AtomicLong()
|
def totalCount = new AtomicLong()
|
||||||
@@ -143,6 +145,7 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
map.location = loc
|
map.location = loc
|
||||||
}
|
}
|
||||||
map.source = "FJW"
|
map.source = "FJW"
|
||||||
|
map.township = dist.township
|
||||||
return map
|
return map
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,17 +174,18 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
|
|
||||||
|
|
||||||
def poiData(city){
|
def poiData(city){
|
||||||
|
def townMap = CommonDatautils.findTownRange(city)
|
||||||
def data = fullCityMap.get(city)
|
def data = fullCityMap.get(city)
|
||||||
if(data){
|
if(data){
|
||||||
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
||||||
createPoi(results, data)
|
createPoi(results, data, townMap)
|
||||||
println city +" 完成!"
|
println city +" 完成!"
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def createPoi(results, data){
|
def createPoi(results, data, townMap){
|
||||||
def rets = new CopyOnWriteArrayList()
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
|
||||||
def totalCount = new AtomicLong()
|
def totalCount = new AtomicLong()
|
||||||
@@ -208,6 +212,15 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
map.cat = dist.cat
|
map.cat = dist.cat
|
||||||
map.location = dist.location
|
map.location = dist.location
|
||||||
map.source = "FJW"
|
map.source = "FJW"
|
||||||
|
|
||||||
|
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||||
|
if(!map.township){
|
||||||
|
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||||
|
if(gaodeMap && gaodeMap.township){
|
||||||
|
map.township = gaodeMap.township
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println map.township
|
||||||
//往100 上同步
|
//往100 上同步
|
||||||
updateData(map)
|
updateData(map)
|
||||||
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
||||||
@@ -242,9 +255,9 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}else{
|
}else if(d.level == "区域"){
|
||||||
if(d.city == "省直辖县级行政区划" && d.district){
|
if(d.administer == "省直辖县级行政区划"){
|
||||||
fjwcity = RegionData.formatCity(d.district)
|
fjwcity = RegionData.formatCity(d.city)
|
||||||
if(fjwcity){
|
if(fjwcity){
|
||||||
d.city = d.district
|
d.city = d.district
|
||||||
}
|
}
|
||||||
@@ -305,4 +318,36 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
|
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def getTownshipInfo(def glocation){
|
||||||
|
def township = null
|
||||||
|
def district = null
|
||||||
|
def jsonObject = null
|
||||||
|
def stop = false
|
||||||
|
def count = 0
|
||||||
|
while(!stop){
|
||||||
|
if(glocation){
|
||||||
|
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||||
|
if(jsonObject && jsonObject.regeocode) {
|
||||||
|
def addressComponent = jsonObject.regeocode.addressComponent
|
||||||
|
if(addressComponent){
|
||||||
|
township = addressComponent.township
|
||||||
|
district = addressComponent.district
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(district || count>10){
|
||||||
|
stop = true
|
||||||
|
}else{
|
||||||
|
count ++
|
||||||
|
Thread.sleep(3000)
|
||||||
|
print "."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def map = [:]
|
||||||
|
map.district = district
|
||||||
|
map.township = township
|
||||||
|
return map;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
|||||||
|
|
||||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||||
def body = data.subList(1, data.size())
|
def body = data.subList(1, data.size())
|
||||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
def nThread = Runtime.getRuntime().availableProcessors() * 3
|
||||||
println nThread
|
println nThread
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||||
final AtomicInteger counter = new AtomicInteger(0)
|
final AtomicInteger counter = new AtomicInteger(0)
|
||||||
|
|||||||
Reference in New Issue
Block a user