This commit is contained in:
2023-08-17 18:10:07 +08:00
parent 5fc4d9fd4e
commit 19e4d20ed4
2 changed files with 213 additions and 0 deletions
@@ -1,8 +1,10 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper
@@ -32,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
class CommonDatautils {
@@ -252,6 +255,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","华润佘山九里二期")
DBObject keys = new BasicDBObject();
if(fields){
@@ -4061,5 +4065,68 @@ class CommonDatautils {
return results
}
static def httpSaveData(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
doPost(newUrl,header, params, true, "utf-8")
}
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setUseCaches(false);
conn.setConnectTimeout(5000);
conn.setReadTimeout(5000);
conn.setRequestProperty("User-Agent", UserAgents.get());
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
conn.setRequestProperty(head, headers.get(head));
}
} else {
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
}
if (isJson) {
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
param = JacksonUtil.beanToJson(params)
}
conn.connect();
//println param.toString()
//输出流
if (StringUtils.isNotEmpty(param)) {
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
mWriter.write(param);
mWriter.flush();
mWriter.close();
}
if (conn.getHeaderField("Content-Encoding") != null
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
} else {
inStream = new InputStreamReader(conn.getInputStream(), charset);
}
char[] bf = new char[1024];
int len;
while ((len = inStream.read(bf)) != -1) {
buffer.append(bf, 0, len);
}
} catch (Exception e) {
e.printStackTrace()
}
return buffer.toString();
}
}
@@ -0,0 +1,146 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class SeaweedDataSynch {
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new SeaweedDataSynch()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
index++
updating(c, "district")
updating(c, "office")
updating(c, "shop")
println index +" "+ c + " Íê³É"
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = createData(dist)
def newUrl = targetUrl +"_doc/"+map.id
CommonDatautils.httpSaveData(newUrl, map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(dist){
def map = [:]
map.id = dist._id.toString()
map.city = dist.city
map.region = dist.region
map.name = dist.name
map.address = dist.address
map.block = dist.block
map.alias= dist.alias
map.addr_alias = dist.addr_alias
map.category = dist.cat
map.estate_type = dist.estate_type2
map.status = dist.status
map.township = dist.township
if(dist.lng){
def loc = [:]
loc.lon = dist.lng/10000000000.0
loc.lat = dist.lat/10000000000.0
map.location = loc
}
//¸ßµÂת°Ù¶È
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = baiduMap.lng
location.lat = baiduMap.lat
map.bd_location = location
}
//¸ßµÂתWGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = gpsLoactionMap.lng
gpslocation.lat = gpsLoactionMap.lat
map.gps_location = gpslocation
}
map.build_type = dist.build_type
map.households = dist.household_count
map.committee = dist.committee
if(dist.estate_charge && dist.estate_charge.toString().isNumber()){
map.property_fee = dist.estate_charge.toString().toFloat()
}
if(dist.car_site_count && dist.car_site_count.toString().isNumber()){
map.car_site_count = dist.car_site_count.toString().toInteger()
}
if(dist.property_life && dist.property_life.toString().isNumber()){
map.property_life = dist.property_life.toString().toInteger()
}
map.house_property = dist.house_property
if(dist.build_area && dist.build_area.toString().isNumber()){
map.build_area = dist.build_area.toString().toDouble().toInteger()
}
if(dist.all_area && dist.all_area.toString().isNumber()){
map.floor_area = dist.all_area.toString().toDouble().toInteger()
}
map.build_structure = dist.build_structure
map.developer = dist.developer
map.property_company = dist.estate_company
map.plot_ratio = dist.cubage_percent
map.landscaping_ratio = dist.virescence_percent
if(dist.loudong_count && dist.loudong_count.toString().isNumber()){
map.building_count = dist.loudong_count.toString().toInteger()
}
map.schools = dist.schools
map.main_house_type = dist.main_type
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
map.main_house_area = dist.main_type_area.toString().toFloat()
}
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
return map
}
}