diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index 1d637247d..e6812a2e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj +import com.fangjia.script.util.ElasticUtils import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet @@ -18,25 +19,24 @@ class EsSearchUtils { static def findPois(def city, def region, def name){ final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1") - sqlBuffer.append(" and cityname = '" + city + "'") - sqlBuffer.append(" and adname = '" + region + "'") - sqlBuffer.append(" and name = '" + name + "'") - - ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString()) def rets = [] - while (rs.next()) { - def m = [:] - m.city = rs.getString("cityname") - m.region = rs.getString("adname") - m.name = rs.getString("name") - m.address = rs.getString("address") - m.type = rs.getString("type") - m.code = rs.getString("typecode") - m.lng = rs.getString("location.lon") - m.lat = rs.getString("location.lat") - rets << m - } + ElasticUtils.executeSqlQuery(sql,{ data-> + def m = [:] + m.city = data.cityname + m.region = data.adname + m.name = data.name + m.address = data.address + m.type = data.type + m.code = data.typecode + + def location = data.location + m.lng = location.lon + m.lat = location.lat + rets << m + }) + return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy new file mode 100644 index 000000000..d0877a9a0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy @@ -0,0 +1,110 @@ +package com.fangjia.script.gj.village + +import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + +def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + +def filePath = "D:\\数据\\涉农\\县.xlsx" +ExcelUtils.multiThreadingModify(filePath, { row -> +// if(!row['D']){ +// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") +// if(rets){ +// def obj = findOneByName(rets, row['C']+"人民政府") +// if(obj){ +// row['D'] = obj.lng +// row['E'] = obj.lat +// println row['C'] + " "+obj +// } +// } +// } + +// if(row['D'] && !row['F']){ +// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL) +// def map = ["lon":row['D'], "lat":row['E']] +// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap) +// row['F'] = township +// } + + if(row['F'] && !row['K']){ + def q = new BasicDBObject() + q.put("city_name", row['B']) + q.put("area_name", row['C']) + def streetName = row['I'] + q.put("street_name", streetName) + def count = COMMUNITY_CONN.count(q); + row['K'] = count +// row['H'] = streetName + } +// def gaodeMap = getTownshipInfo(row['D']+","+row['E']) +// if(gaodeMap && gaodeMap.township){ +// row['I'] = gaodeMap.township +// } + +}) + + +def getTownshipInfo(def glocation){ + def township = null + def district = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.district = district + map.township = township + return map; +} + + +def findOneByName(def rets, name){ + def obj = null + for(def r : rets){ + if(r.name == name){ + obj = r + break; + } + } + return obj +} + + +def findTownRange(city, TOWN_POLYLINECOLL){ + def q = new BasicDBObject() + q.put("city",city) + def dbcursor = TOWN_POLYLINECOLL.find(q) + def townMap = [:] + for(def d : dbcursor){ + def pointList = InsidePolygon.createPoint3(d.polyline); + townMap.put(d.fjw_region +"##"+ d.town, pointList) + } + return townMap +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 104e28def..b24c6f690 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -63,22 +63,59 @@ class SuggestionDataSynch extends BaseBatch{ mcaDistMap = queryDistNameCity() for (def c : citys) { + def allRets = [] index++ // if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){ - updating(c, "district") - updating(c, "office") - updating(c, "shop") + def ret1 = updating(c, "district") + def ret2 = updating(c, "office") + def ret3 = updating(c, "shop") // } // if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) { - poiData(c) + def ret4 = poiData(c) // } - println index + " " + c + " 完成" + allRets.addAll(ret1) + allRets.addAll(ret2) + allRets.addAll(ret3) + allRets.addAll(ret4) + println index + " " + c + " 完成" + saveToMG(c, allRets) } } + + + def saveToMG(city, rows){ + + def data = fullCityMap.get(city) + def q = new BasicDBObject("city", data.city) + def count = SUGGESTION_CONN.count(q) + if(count>0){ + SUGGESTION_CONN.remove(q) + } + + def rets = [] + for(def i=0;i + obj.put(k,v) + } + rets << obj + if(i%50000==0){ + SUGGESTION_CONN.insert(rets) + rets.clear() + } + } + if(rets.size()>0){ + SUGGESTION_CONN.insert(rets) + rets.clear() + } + } + + def updating(city, cat){ def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"]) def rets = new CopyOnWriteArrayList() @@ -98,7 +135,8 @@ class SuggestionDataSynch extends BaseBatch{ def map = createData(dist) //往100 上同步 - updateData(map) +// updateData(map) + rets << map println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id } } catch (Exception e) { @@ -227,7 +265,8 @@ class SuggestionDataSynch extends BaseBatch{ } println map.township //往100 上同步 - updateData(map) +// updateData(map) + rets << map println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id } } catch (Exception e) {