update
This commit is contained in:
+67
@@ -1,8 +1,10 @@
|
|||||||
package com.fangjia.script.commData
|
package com.fangjia.script.commData
|
||||||
|
|
||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.httpclient.UserAgents
|
||||||
import com.fangjia.script.util.DbUtils
|
import com.fangjia.script.util.DbUtils
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
|
import com.fangjia.script.util.JacksonUtil
|
||||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||||
import groovy.json.JsonSlurper
|
import groovy.json.JsonSlurper
|
||||||
|
|
||||||
@@ -32,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders
|
|||||||
|
|
||||||
import java.util.regex.Matcher
|
import java.util.regex.Matcher
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
import java.util.zip.GZIPInputStream
|
||||||
|
|
||||||
class CommonDatautils {
|
class CommonDatautils {
|
||||||
|
|
||||||
@@ -252,6 +255,7 @@ class CommonDatautils {
|
|||||||
q.put("visible", 0);
|
q.put("visible", 0);
|
||||||
q.put("cat", cat);
|
q.put("cat", cat);
|
||||||
q.put("city", city);
|
q.put("city", city);
|
||||||
|
// q.put("name","华润佘山九里二期")
|
||||||
|
|
||||||
DBObject keys = new BasicDBObject();
|
DBObject keys = new BasicDBObject();
|
||||||
if(fields){
|
if(fields){
|
||||||
@@ -4061,5 +4065,68 @@ class CommonDatautils {
|
|||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def httpSaveData(def newUrl,def params){
|
||||||
|
def header = [:]
|
||||||
|
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
|
||||||
|
doPost(newUrl,header, params, true, "utf-8")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||||
|
StringBuilder buffer = new StringBuilder();
|
||||||
|
try{
|
||||||
|
URL url = new URL(targetUrl);
|
||||||
|
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||||
|
conn.setRequestMethod("POST");
|
||||||
|
conn.setDoOutput(true);
|
||||||
|
conn.setUseCaches(false);
|
||||||
|
conn.setConnectTimeout(5000);
|
||||||
|
conn.setReadTimeout(5000);
|
||||||
|
conn.setRequestProperty("User-Agent", UserAgents.get());
|
||||||
|
String param = null;
|
||||||
|
BufferedWriter mWriter = null;
|
||||||
|
InputStreamReader inStream = null;
|
||||||
|
|
||||||
|
|
||||||
|
if (headers != null && headers.keySet().size() > 0) {
|
||||||
|
for (String head : headers.keySet()) {
|
||||||
|
conn.setRequestProperty(head, headers.get(head));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isJson) {
|
||||||
|
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
|
||||||
|
param = JacksonUtil.beanToJson(params)
|
||||||
|
}
|
||||||
|
|
||||||
|
conn.connect();
|
||||||
|
|
||||||
|
//println param.toString()
|
||||||
|
//输出流
|
||||||
|
if (StringUtils.isNotEmpty(param)) {
|
||||||
|
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
|
||||||
|
mWriter.write(param);
|
||||||
|
mWriter.flush();
|
||||||
|
mWriter.close();
|
||||||
|
}
|
||||||
|
if (conn.getHeaderField("Content-Encoding") != null
|
||||||
|
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
|
||||||
|
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
|
||||||
|
} else {
|
||||||
|
inStream = new InputStreamReader(conn.getInputStream(), charset);
|
||||||
|
}
|
||||||
|
char[] bf = new char[1024];
|
||||||
|
int len;
|
||||||
|
while ((len = inStream.read(bf)) != -1) {
|
||||||
|
buffer.append(bf, 0, len);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
}
|
||||||
|
return buffer.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+146
@@ -0,0 +1,146 @@
|
|||||||
|
package com.fangjia.script.gj.xiamen
|
||||||
|
|
||||||
|
import com.fangjia.core.util.GeoUtils
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.DateUtil
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
class SeaweedDataSynch {
|
||||||
|
|
||||||
|
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new SeaweedDataSynch()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
def index = 0
|
||||||
|
for(def c : citys){
|
||||||
|
index++
|
||||||
|
updating(c, "district")
|
||||||
|
updating(c, "office")
|
||||||
|
updating(c, "shop")
|
||||||
|
println index +" "+ c + " Íê³É"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def updating(city, cat){
|
||||||
|
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area"])
|
||||||
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
for(def i=0;i<dists.size();i++){
|
||||||
|
final def dist = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
|
||||||
|
def map = createData(dist)
|
||||||
|
def newUrl = targetUrl +"_doc/"+map.id
|
||||||
|
CommonDatautils.httpSaveData(newUrl, map)
|
||||||
|
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
println dist.city +" "+dist.name
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
def createData(dist){
|
||||||
|
def map = [:]
|
||||||
|
map.id = dist._id.toString()
|
||||||
|
map.city = dist.city
|
||||||
|
map.region = dist.region
|
||||||
|
map.name = dist.name
|
||||||
|
map.address = dist.address
|
||||||
|
map.block = dist.block
|
||||||
|
map.alias= dist.alias
|
||||||
|
map.addr_alias = dist.addr_alias
|
||||||
|
map.category = dist.cat
|
||||||
|
map.estate_type = dist.estate_type2
|
||||||
|
map.status = dist.status
|
||||||
|
map.township = dist.township
|
||||||
|
if(dist.lng){
|
||||||
|
def loc = [:]
|
||||||
|
loc.lon = dist.lng/10000000000.0
|
||||||
|
loc.lat = dist.lat/10000000000.0
|
||||||
|
map.location = loc
|
||||||
|
}
|
||||||
|
|
||||||
|
//¸ßµÂת°Ù¶È
|
||||||
|
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||||
|
if(baiduMap){
|
||||||
|
def location = [:]
|
||||||
|
location.lon = baiduMap.lng
|
||||||
|
location.lat = baiduMap.lat
|
||||||
|
map.bd_location = location
|
||||||
|
}
|
||||||
|
|
||||||
|
//¸ßµÂתWGS84
|
||||||
|
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||||
|
if(gpsLoactionMap){
|
||||||
|
def gpslocation = [:]
|
||||||
|
gpslocation.lon = gpsLoactionMap.lng
|
||||||
|
gpslocation.lat = gpsLoactionMap.lat
|
||||||
|
map.gps_location = gpslocation
|
||||||
|
}
|
||||||
|
|
||||||
|
map.build_type = dist.build_type
|
||||||
|
map.households = dist.household_count
|
||||||
|
map.committee = dist.committee
|
||||||
|
if(dist.estate_charge && dist.estate_charge.toString().isNumber()){
|
||||||
|
map.property_fee = dist.estate_charge.toString().toFloat()
|
||||||
|
}
|
||||||
|
|
||||||
|
if(dist.car_site_count && dist.car_site_count.toString().isNumber()){
|
||||||
|
map.car_site_count = dist.car_site_count.toString().toInteger()
|
||||||
|
}
|
||||||
|
if(dist.property_life && dist.property_life.toString().isNumber()){
|
||||||
|
map.property_life = dist.property_life.toString().toInteger()
|
||||||
|
}
|
||||||
|
|
||||||
|
map.house_property = dist.house_property
|
||||||
|
|
||||||
|
if(dist.build_area && dist.build_area.toString().isNumber()){
|
||||||
|
map.build_area = dist.build_area.toString().toDouble().toInteger()
|
||||||
|
}
|
||||||
|
|
||||||
|
if(dist.all_area && dist.all_area.toString().isNumber()){
|
||||||
|
map.floor_area = dist.all_area.toString().toDouble().toInteger()
|
||||||
|
}
|
||||||
|
|
||||||
|
map.build_structure = dist.build_structure
|
||||||
|
map.developer = dist.developer
|
||||||
|
map.property_company = dist.estate_company
|
||||||
|
map.plot_ratio = dist.cubage_percent
|
||||||
|
map.landscaping_ratio = dist.virescence_percent
|
||||||
|
if(dist.loudong_count && dist.loudong_count.toString().isNumber()){
|
||||||
|
map.building_count = dist.loudong_count.toString().toInteger()
|
||||||
|
}
|
||||||
|
|
||||||
|
map.schools = dist.schools
|
||||||
|
map.main_house_type = dist.main_type
|
||||||
|
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
|
||||||
|
map.main_house_area = dist.main_type_area.toString().toFloat()
|
||||||
|
}
|
||||||
|
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||||
|
return map
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user