update
This commit is contained in:
+67
@@ -1,8 +1,10 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
@@ -32,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class CommonDatautils {
|
||||
|
||||
@@ -252,6 +255,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","华润佘山九里二期")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -4061,5 +4065,68 @@ class CommonDatautils {
|
||||
return results
|
||||
}
|
||||
|
||||
static def httpSaveData(def newUrl,def params){
|
||||
def header = [:]
|
||||
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
|
||||
doPost(newUrl,header, params, true, "utf-8")
|
||||
}
|
||||
|
||||
|
||||
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
try{
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setDoOutput(true);
|
||||
conn.setUseCaches(false);
|
||||
conn.setConnectTimeout(5000);
|
||||
conn.setReadTimeout(5000);
|
||||
conn.setRequestProperty("User-Agent", UserAgents.get());
|
||||
String param = null;
|
||||
BufferedWriter mWriter = null;
|
||||
InputStreamReader inStream = null;
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
conn.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
} else {
|
||||
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
|
||||
}
|
||||
|
||||
if (isJson) {
|
||||
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
|
||||
param = JacksonUtil.beanToJson(params)
|
||||
}
|
||||
|
||||
conn.connect();
|
||||
|
||||
//println param.toString()
|
||||
//输出流
|
||||
if (StringUtils.isNotEmpty(param)) {
|
||||
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
|
||||
mWriter.write(param);
|
||||
mWriter.flush();
|
||||
mWriter.close();
|
||||
}
|
||||
if (conn.getHeaderField("Content-Encoding") != null
|
||||
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
|
||||
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
|
||||
} else {
|
||||
inStream = new InputStreamReader(conn.getInputStream(), charset);
|
||||
}
|
||||
char[] bf = new char[1024];
|
||||
int len;
|
||||
while ((len = inStream.read(bf)) != -1) {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class SeaweedDataSynch {
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new SeaweedDataSynch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index++
|
||||
updating(c, "district")
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
println index +" "+ c + " Íê³É"
|
||||
}
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def map = createData(dist)
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
CommonDatautils.httpSaveData(newUrl, map)
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map.id = dist._id.toString()
|
||||
map.city = dist.city
|
||||
map.region = dist.region
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
map.block = dist.block
|
||||
map.alias= dist.alias
|
||||
map.addr_alias = dist.addr_alias
|
||||
map.category = dist.cat
|
||||
map.estate_type = dist.estate_type2
|
||||
map.status = dist.status
|
||||
map.township = dist.township
|
||||
if(dist.lng){
|
||||
def loc = [:]
|
||||
loc.lon = dist.lng/10000000000.0
|
||||
loc.lat = dist.lat/10000000000.0
|
||||
map.location = loc
|
||||
}
|
||||
|
||||
//¸ßµÂת°Ù¶È
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = baiduMap.lng
|
||||
location.lat = baiduMap.lat
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//¸ßµÂתWGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = gpsLoactionMap.lng
|
||||
gpslocation.lat = gpsLoactionMap.lat
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
map.build_type = dist.build_type
|
||||
map.households = dist.household_count
|
||||
map.committee = dist.committee
|
||||
if(dist.estate_charge && dist.estate_charge.toString().isNumber()){
|
||||
map.property_fee = dist.estate_charge.toString().toFloat()
|
||||
}
|
||||
|
||||
if(dist.car_site_count && dist.car_site_count.toString().isNumber()){
|
||||
map.car_site_count = dist.car_site_count.toString().toInteger()
|
||||
}
|
||||
if(dist.property_life && dist.property_life.toString().isNumber()){
|
||||
map.property_life = dist.property_life.toString().toInteger()
|
||||
}
|
||||
|
||||
map.house_property = dist.house_property
|
||||
|
||||
if(dist.build_area && dist.build_area.toString().isNumber()){
|
||||
map.build_area = dist.build_area.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
if(dist.all_area && dist.all_area.toString().isNumber()){
|
||||
map.floor_area = dist.all_area.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
map.build_structure = dist.build_structure
|
||||
map.developer = dist.developer
|
||||
map.property_company = dist.estate_company
|
||||
map.plot_ratio = dist.cubage_percent
|
||||
map.landscaping_ratio = dist.virescence_percent
|
||||
if(dist.loudong_count && dist.loudong_count.toString().isNumber()){
|
||||
map.building_count = dist.loudong_count.toString().toInteger()
|
||||
}
|
||||
|
||||
map.schools = dist.schools
|
||||
map.main_house_type = dist.main_type
|
||||
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
|
||||
map.main_house_area = dist.main_type_area.toString().toFloat()
|
||||
}
|
||||
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
return map
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user