diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8ee79c56c..b81154fb8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,8 +1,10 @@ package com.fangjia.script.commData import com.fangjia.core.RegionData +import com.fangjia.httpclient.UserAgents import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.JacksonUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import groovy.json.JsonSlurper @@ -32,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders import java.util.regex.Matcher import java.util.regex.Pattern +import java.util.zip.GZIPInputStream class CommonDatautils { @@ -252,6 +255,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); +// q.put("name","华润佘山九里二期") DBObject keys = new BasicDBObject(); if(fields){ @@ -4061,5 +4065,68 @@ class CommonDatautils { return results } + static def httpSaveData(def newUrl,def params){ + def header = [:] + header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=") + doPost(newUrl,header, params, true, "utf-8") + } + + + static def doPost(def targetUrl, def headers, def params, def isJson, def charset){ + StringBuilder buffer = new StringBuilder(); + try{ + URL url = new URL(targetUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setUseCaches(false); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + conn.setRequestProperty("User-Agent", UserAgents.get()); + String param = null; + BufferedWriter mWriter = null; + InputStreamReader inStream = null; + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + conn.setRequestProperty(head, headers.get(head)); + } + } else { + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" ); + } + + if (isJson) { + conn.setRequestProperty("Content-Type", "application/json; charset=" + charset); + param = JacksonUtil.beanToJson(params) + } + + conn.connect(); + + //println param.toString() + //输出流 + if (StringUtils.isNotEmpty(param)) { + mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset)); + mWriter.write(param); + mWriter.flush(); + mWriter.close(); + } + if (conn.getHeaderField("Content-Encoding") != null + && conn.getHeaderField("Content-Encoding").equals("gzip")) { + inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset); + } else { + inStream = new InputStreamReader(conn.getInputStream(), charset); + } + char[] bf = new char[1024]; + int len; + while ((len = inStream.read(bf)) != -1) { + buffer.append(bf, 0, len); + } + } catch (Exception e) { + e.printStackTrace() + } + return buffer.toString(); + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy new file mode 100644 index 000000000..8efdcd037 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -0,0 +1,146 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtil +import com.fangjia.script.util.DateUtils + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class SeaweedDataSynch { + + static def targetUrl = "http://192.168.0.12:9200/seaweed/" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + static main(args) { + def task = new SeaweedDataSynch() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def c : citys){ + index++ + updating(c, "district") + updating(c, "office") + updating(c, "shop") + println index +" "+ c + " 完成" + } + } + + def updating(city, cat){ + def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area"]) + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i