Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
@@ -537,6 +537,12 @@
|
||||
<version>1.70</version> <!-- 确保使用最新版本 -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
<artifactId>jts-core</artifactId>
|
||||
<version>1.18.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>com.fangjia</groupId>-->
|
||||
<!-- <artifactId>fangjia-open</artifactId>-->
|
||||
|
||||
@@ -78,7 +78,7 @@ class PoseidonOnce {
|
||||
// DBCursor cur = coll.find()
|
||||
// println cur.count()
|
||||
String url = "http://www.fangjia.com";
|
||||
DBCollection coll = MongoUtils.getCollectionByName("fangjia", "test2", "192.168.0.188")
|
||||
DBCollection coll = MongoUtils.getCollectionByName("fangjia", "FiveLevelTest", "192.168.0.188")
|
||||
// for(int i=426000;i<1000000;i++) {
|
||||
// String s = url + i + "_" + url.hashCode()
|
||||
// DBObject dbo = new BasicDBObject()
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+1
-2
@@ -71,8 +71,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
|
||||
+65
-2
@@ -30,8 +30,11 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
|
||||
"淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12","房多多":"13","我爱我家":"14",
|
||||
"禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20",
|
||||
"乐有家":"21","幸福里":"22","中国房价行情":"23","常州住房":"24"
|
||||
]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -263,6 +266,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("csjj_esf_price")){
|
||||
price_map.csjj_avg_price = third.get("csjj_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("leyoujia_esf_price")){
|
||||
price_map.leyoujia_avg_price = third.get("leyoujia_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("c21_esf_price")){
|
||||
price_map.c21_avg_price = third.get("c21_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("jiwu_esf_price")){
|
||||
price_map.jiwu_avg_price = third.get("jiwu_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("xingfuli_esf_price")){
|
||||
price_map.xingfuli_avg_price = third.get("xingfuli_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("zwfc_esf_price")){
|
||||
price_map.zwfc_avg_price = third.get("zwfc_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("creprice_esf_price")){
|
||||
price_map.creprice_avg_price = third.get("creprice_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -313,6 +340,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房产超市"
|
||||
}else if(k=="csjj_avg_price"){
|
||||
web = "城市居家"
|
||||
}else if(k=="leyoujia_avg_price"){
|
||||
web = "乐有家"
|
||||
}else if(k=="c21_avg_price"){
|
||||
web = "C21"
|
||||
}else if(k=="jiwu_avg_price"){
|
||||
web = "吉屋网"
|
||||
}else if(k=="xingfuli_avg_price"){
|
||||
web = "幸福里"
|
||||
}else if(k=="creprice_avg_price"){
|
||||
web = "中国房价行情"
|
||||
}else if(k=="zwfc_avg_price"){
|
||||
web = "常州住房"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -603,6 +642,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
|
||||
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("leyoujia_esf_price") && db.get("leyoujia_esf_price") && db.get("leyoujia_esf_price")>3000){
|
||||
dataMap.put("leyoujia_esf_price", db.get("leyoujia_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("c21_esf_price") && db.get("c21_esf_price") && db.get("c21_esf_price")>3000){
|
||||
dataMap.put("c21_esf_price", db.get("c21_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("jiwu_esf_price") && db.get("jiwu_esf_price") && db.get("jiwu_esf_price")>3000){
|
||||
dataMap.put("jiwu_esf_price", db.get("jiwu_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("xingfuli_esf_price") && db.get("xingfuli_esf_price") && db.get("xingfuli_esf_price")>3000){
|
||||
dataMap.put("xingfuli_esf_price", db.get("xingfuli_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("creprice_esf_price") && db.get("creprice_esf_price") && db.get("creprice_esf_price")>3000){
|
||||
dataMap.put("creprice_esf_price", db.get("creprice_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("zwfc_esf_price") && db.get("zwfc_esf_price") && db.get("zwfc_esf_price")>3000){
|
||||
dataMap.put("zwfc_esf_price", db.get("zwfc_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+37
-8
@@ -103,6 +103,8 @@ class CommonDatautils {
|
||||
|
||||
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -597,9 +599,9 @@ class CommonDatautils {
|
||||
keys.put("region", 1);
|
||||
keys.put("name", 1);
|
||||
keys.put("_id", 1);
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
|
||||
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
|
||||
Iterator<DBObject> iters = cursor.iterator()
|
||||
def result = [];
|
||||
def result = new HashSet();
|
||||
try{
|
||||
while(iters.hasNext()) {
|
||||
DBObject obj = iters.next()
|
||||
@@ -4209,19 +4211,21 @@ class CommonDatautils {
|
||||
|
||||
|
||||
static def findAllNewPoiDatas(city){
|
||||
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
|
||||
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
index++
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
|
||||
// }
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def findOneThirdNetWorkOffice(city, region, name){
|
||||
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
|
||||
def map = [:]
|
||||
@@ -4965,6 +4969,20 @@ class CommonDatautils {
|
||||
return regionMap
|
||||
}
|
||||
|
||||
static def queryRangeByCat(cat){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type",cat)
|
||||
q.put("shape",new BasicDBObject('$exists', true))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city_short",1)
|
||||
keys.put("district_short",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList()
|
||||
}
|
||||
|
||||
|
||||
static def queryAllRegionNameByCity(){
|
||||
|
||||
@@ -5135,17 +5153,22 @@ class CommonDatautils {
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def distName = null
|
||||
if(d.mzb_district_name){
|
||||
def list = townMap.get(d.mzb_district_name)
|
||||
distName = d.mzb_district_name
|
||||
}else{
|
||||
distName = d.district
|
||||
}
|
||||
if(distName){
|
||||
def list = townMap.get(distName)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(d.mzb_district_name, list)
|
||||
townMap.put(distName, list)
|
||||
}
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
@@ -5515,4 +5538,10 @@ class CommonDatautils {
|
||||
return 0
|
||||
}
|
||||
|
||||
static def findBusinessByCity(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("short_city", city)
|
||||
return bk_business_coll.find(q).toList()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+15
-15
@@ -343,21 +343,21 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
def totalDbhouseloudongCount = HOUSE_LOUDONG_CONN.count(findObj)
|
||||
datas << ['', totalDistrictCount, totalLoudongCount, df.format(totalLoudongCount * 1.0/ totalDistrictCount), totalFanghaoCount, df.format(totalFanghaoCount * 1.0/ totalDistrictCount), totalDbLoudongCount, totalDbhouseloudongCount]
|
||||
|
||||
ExcelUtils.write(fileName, datas, resultTitle)
|
||||
String id = ObjectId.get().toString()
|
||||
BasicDBObject file = new BasicDBObject()
|
||||
file.put("t", "upload")
|
||||
file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
||||
file.put("c_date", new Date())
|
||||
file.put("_id", idGenerator(id, "o"))
|
||||
file.put("data", new File(fileName).getBytes())
|
||||
fsDB.save(file)
|
||||
def href = DataUtil.encodePicUrl(id, "o", "upload")
|
||||
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
|
||||
receivers.each {receiver->
|
||||
jms.sendMail(receiver, resultTitle, href)
|
||||
}
|
||||
new File(fileName).deleteOnExit()
|
||||
// ExcelUtils.write(fileName, datas, resultTitle)
|
||||
// String id = ObjectId.get().toString()
|
||||
// BasicDBObject file = new BasicDBObject()
|
||||
// file.put("t", "upload")
|
||||
// file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
||||
// file.put("c_date", new Date())
|
||||
// file.put("_id", idGenerator(id, "o"))
|
||||
// file.put("data", new File(fileName).getBytes())
|
||||
// fsDB.save(file)
|
||||
// def href = DataUtil.encodePicUrl(id, "o", "upload")
|
||||
// href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
|
||||
// receivers.each {receiver->
|
||||
// jms.sendMail(receiver, resultTitle, href)
|
||||
// }
|
||||
// new File(fileName).deleteOnExit()
|
||||
}
|
||||
|
||||
private String idGenerator(String url, String size) {
|
||||
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
package com.fangjia.script.dealPrice
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
/**
|
||||
* 成交同步206,按日同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class SynchDealPriceToEs extends BaseBatch{
|
||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDealPriceToEs(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
|
||||
def q = new BasicDBObject()
|
||||
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcursor = dealPriceConn.find(q)
|
||||
println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
|
||||
def ret = []
|
||||
for(def o : dbcursor){
|
||||
ret << o
|
||||
}
|
||||
save(ret)
|
||||
|
||||
// def batchsize = 2000
|
||||
// def citys = getCityCount()
|
||||
// citys.each{city,count->
|
||||
// if(city instanceof String){
|
||||
// for(def year = 1999;year<=2025;year++){
|
||||
// def q = new BasicDBObject("city", city)
|
||||
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
|
||||
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
|
||||
// println city +" "+ dbcursor.count()
|
||||
// def ret = []
|
||||
// for(def o : dbcursor){
|
||||
// ret << o
|
||||
// }
|
||||
// save(ret)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
def getCityCount(){
|
||||
|
||||
BasicDBObject match = null;
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
|
||||
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
|
||||
groupFields.put("_id", new BasicDBObject("city",'$city'))
|
||||
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
|
||||
// 执行聚合查询
|
||||
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
|
||||
def map = [:]
|
||||
// 遍历结果
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
if(obj.get("city")) {
|
||||
int count = (Integer) result.get("count");
|
||||
map.put(obj.get("city"), count)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 500;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.206")
|
||||
|
||||
for(def map : rows){
|
||||
// def map = rows.get(key)
|
||||
def data_id = map._id.toString()
|
||||
map.remove("_id")
|
||||
request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -909,14 +909,18 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
static def getNumber(def input) {
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
try{
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
}
|
||||
}catch(Exception e){
|
||||
println input+" error"
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
|
||||
class ProxyHttpUtils {
|
||||
|
||||
static String proxyHost = 'tunnel2.qg.net';
|
||||
static Integer proxyPort = 14186;
|
||||
static String proxyUsername = '113BBC06';
|
||||
static String proxyPassword = '3B2FBD05B0AF';
|
||||
|
||||
static String getProxyResult(targetUrl, data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
String jsonData = new JsonOutput().toJson(data);
|
||||
// 创建代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
|
||||
// 创建 URL 对象
|
||||
URL url = new URL(targetUrl);
|
||||
|
||||
// 创建连接对象
|
||||
URLConnection connection = url.openConnection(proxy);
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST");
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true);
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json");
|
||||
|
||||
// 发送请求参数
|
||||
OutputStream outputStream = connection.getOutputStream();
|
||||
outputStream.write(jsonData.getBytes("UTF-8"));
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 读取响应
|
||||
int responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line);
|
||||
}
|
||||
reader.close();
|
||||
return response.toString();
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.gj.coresite
|
||||
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import java.io.OutputStream
|
||||
|
||||
// 目标服务器的URL
|
||||
def targetUrlString = "http://data.zz.baidu.com/urls?site=sh.fangjia.com&token=UaKTemkcQ6I4YvSA"
|
||||
|
||||
// 需要POST的URL数据
|
||||
def postData = "https://sh.fangjia.com/zoushi?cityName=%E4%B8%8A%E6%B5%B7&keyword=%E5%A4%A9%E5%B1%B1%E6%96%B0%E8%8B%91"
|
||||
|
||||
// 创建URL对象
|
||||
URL url = new URL(targetUrlString)
|
||||
|
||||
// 打开连接
|
||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection()
|
||||
connection.setDoOutput(true) // 表明我们将发送POST请求的输出数据
|
||||
connection.setRequestMethod("POST") // 设置请求方法为POST
|
||||
connection.setRequestProperty("User-Agent", "curl/7.12.1") // 设置User-Agent,尽管这在实际应用中可能并非必需
|
||||
connection.setRequestProperty("Content-Type", "text/plain") // 设置内容类型,因为我们将发送纯文本数据
|
||||
connection.setRequestProperty("Content-Length", Integer.toString(postData.getBytes().length)) // 设置内容长度
|
||||
|
||||
// 发送POST数据
|
||||
//try (OutputStream os = connection.getOutputStream()) {
|
||||
// byte[] input = postData.getBytes("utf-8")
|
||||
// os.write(input, 0, input.length)
|
||||
//}
|
||||
outputStream = connection.getOutputStream();
|
||||
outputStream.write(postData.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 获取响应码
|
||||
int responseCode = connection.getResponseCode()
|
||||
println "POST Response Code :: $responseCode"
|
||||
|
||||
// 可选:读取响应内容
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"))
|
||||
StringBuilder response = new StringBuilder()
|
||||
String responseLine
|
||||
while ((responseLine = reader.readLine()) != null) {
|
||||
response.append(responseLine.trim())
|
||||
}
|
||||
println "Response Body :: ${response.toString()}"
|
||||
|
||||
// 关闭连接
|
||||
connection.disconnect()
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.io.FileType
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class AddDevShapeData {
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
def devAreaMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDevShapeData();
|
||||
task.run()
|
||||
// task.streetInfo()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\111开发区\\'
|
||||
def dir = new File(directoryPath)
|
||||
if (dir.isDirectory()) {
|
||||
// dir.eachFileRecurse(FileType.FILES) { file ->
|
||||
// process(file.absolutePath)
|
||||
// }
|
||||
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
|
||||
} else {
|
||||
println "The specified path is not a directory."
|
||||
}
|
||||
}
|
||||
|
||||
def addDevInfo(file){
|
||||
ExcelUtils.multiThreadingModify(file, {d->
|
||||
// if(!d['I']){
|
||||
def name = d['D']
|
||||
def q = new BasicDBObject()
|
||||
q.put('_id',new ObjectId(d['A']))
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
|
||||
if(obj){
|
||||
if(obj.operator){
|
||||
d['H'] = obj.operator
|
||||
}
|
||||
if(obj.operate_result){
|
||||
d['I'] = obj.operate_result
|
||||
}
|
||||
if(obj.m_date){
|
||||
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
// }
|
||||
devAreaMap.each {k,v->
|
||||
if(name.contains(k)){
|
||||
d['M'] = v.area_info
|
||||
d['N'] = v.count
|
||||
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def process(file){
|
||||
def retMap = getNewLocations(file)
|
||||
if(retMap && retMap.gaode.size()>0){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+retMap.name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
|
||||
if(obj){
|
||||
for(def i=0;i<retMap.gaode.size();i++){
|
||||
def udata = new BasicDBObject()
|
||||
def resultMap = [:]
|
||||
def bd_resultMap = [:]
|
||||
|
||||
resultMap.type = "polygon"
|
||||
resultMap.coordinates = [retMap.gaode[i]]
|
||||
|
||||
bd_resultMap.type = "polygon"
|
||||
bd_resultMap.coordinates = [retMap.baidu[i]]
|
||||
if(i>0){
|
||||
obj.removeField("_id")
|
||||
obj.put("create_time", new Date())
|
||||
obj.put("block", retMap.name+"-地块"+i)
|
||||
obj.put("shape", resultMap)
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("bd_shape",bd_resultMap)
|
||||
obj.put("operator", "外包")
|
||||
obj.put( "operate_result" , "已处理")
|
||||
DEVELOPMENT_ZONE_43_COLL.save(obj)
|
||||
}else{
|
||||
udata.put("shape", resultMap)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("bd_shape",bd_resultMap)
|
||||
udata.put("operate_result" , "已处理")
|
||||
udata.put("operator", "外包")
|
||||
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getNewLocations(file){
|
||||
def jsonObj = readJson(file)
|
||||
def features = jsonObj.features
|
||||
def gaode_locations = []
|
||||
def baidu_locations = []
|
||||
def name = null
|
||||
def areaMap = [:]
|
||||
if(features && features.size()==1){
|
||||
name = features[0].properties.Name
|
||||
def coordinates = features[0].geometry.coordinates
|
||||
|
||||
def index = 0
|
||||
println name +" 有 "+coordinates.size()+" 块"
|
||||
|
||||
areaMap.count = coordinates.size()
|
||||
def areas = []
|
||||
if(name.contains("泉州经济技术开发区")){
|
||||
for(def pointList : coordinates){
|
||||
index ++
|
||||
def gaode_coordinates = []
|
||||
def bd_coordinates = []
|
||||
def points = null
|
||||
if(pointList.size()>1){
|
||||
points = pointList
|
||||
}else{
|
||||
points = pointList[0]
|
||||
}
|
||||
double area = com.fangjia.script.util.GeoHelper.calculateArea(points)
|
||||
println name +" 地块"+index + "围栏面积是: ${area} 平方公里"
|
||||
areas << area
|
||||
def locationstr = ""
|
||||
for(def arrs : points){
|
||||
def gaode_map = GeoUtils.gps84ToGcj02(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
gaode_coordinates << [gaode_map.get("lng"), gaode_map.get("lat")]
|
||||
|
||||
//此高德转百度 有偏差
|
||||
def bd_map = GeoUtils.gcj02ToBd09(gaode_map.get("lat"), gaode_map.get("lng"))
|
||||
// bd_coordinates << [bd_map.get("lng"), bd_map.get("lat")]
|
||||
// 高德转百度 通过接口有次数限制
|
||||
// def baidu = BaiDuApiUtils.geoconv(gaode_map.get("lng"), gaode_map.get("lat"), 3)
|
||||
// bd_coordinates << [baidu.x, baidu.y]
|
||||
|
||||
locationstr = arrs[0].toString()+","+arrs[1].toString()
|
||||
def bdpoint = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()), 6)
|
||||
if(bdpoint){
|
||||
bd_coordinates << bdpoint
|
||||
}
|
||||
}
|
||||
//整个一串一起传超过4029会报错
|
||||
// bd_coordinates = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()-1), 6)
|
||||
gaode_locations << gaode_coordinates
|
||||
if(bd_coordinates){
|
||||
baidu_locations << bd_coordinates
|
||||
}else{
|
||||
println name +" 转换有问题"
|
||||
}
|
||||
|
||||
}
|
||||
areaMap.area_info = areas.join("\n")
|
||||
}
|
||||
|
||||
}else{
|
||||
println file +" 有多个"
|
||||
}
|
||||
|
||||
devAreaMap.put(name, areaMap)
|
||||
def retMap = [:]
|
||||
retMap.name = name
|
||||
retMap.gaode = gaode_locations
|
||||
retMap.baidu = baidu_locations
|
||||
return retMap
|
||||
}
|
||||
|
||||
def readJson(def file){
|
||||
def jsonSlurper = new JsonSlurper()
|
||||
def jsonFile = new File(file)
|
||||
def jsonText = jsonFile.text
|
||||
def jsonObj = jsonSlurper.parseText(jsonText)
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
def streetInfo(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx", {d->
|
||||
def count = dev_mapping_CONN.count(new BasicDBObject("name", d['E']).append("type", "乡镇街道"))
|
||||
d['L'] = count
|
||||
})
|
||||
}
|
||||
}
|
||||
+448
@@ -0,0 +1,448 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.json.JsonSlurper
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
//补充库中没有详细地址的小区
|
||||
class AddDistrictAddress extends BaseBatch{
|
||||
|
||||
// def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
// def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
//
|
||||
// DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
// public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
//
|
||||
//
|
||||
// public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
//
|
||||
// def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
// DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
//
|
||||
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictAddress(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// addNoAddressDist() //为不精确的地址通过地图补充地址
|
||||
// mapOtherAddress() //有精确地址的寻找其它地址
|
||||
|
||||
// addressSelect()
|
||||
addrUpdateToDB()
|
||||
}
|
||||
|
||||
|
||||
def addNoAddressDist(){
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def index = 0
|
||||
// def allrets = []
|
||||
// for(def c : citys){
|
||||
// index ++
|
||||
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
|
||||
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
|
||||
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
|
||||
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
|
||||
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
|
||||
//
|
||||
// def rets = []
|
||||
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
|
||||
// for(def d : noRoadInfoDists){
|
||||
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
|
||||
// }
|
||||
// allrets.addAll(rets)
|
||||
// if(allrets.size()>50000){
|
||||
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
|
||||
// allrets.clear()
|
||||
// }
|
||||
// }
|
||||
updateDistAddr()
|
||||
}
|
||||
|
||||
|
||||
def mapOtherAddress(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
def allrets = []
|
||||
for(def c : citys){
|
||||
index ++
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address", "addr_alias"])
|
||||
def radInfoDists = haveRoadInfoDistricts(districts)
|
||||
println c +" "+index +" 原始小区量:"+districts.size() +" 有详细地址量:"+radInfoDists.size()
|
||||
radInfoDists = baiduMatch(c, radInfoDists)
|
||||
radInfoDists = gaodeMatch(c, radInfoDists)
|
||||
|
||||
def rets = []
|
||||
rets << ["id", "城市", "区域", "名称", "地址","地址别名", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
|
||||
for(def d : radInfoDists){
|
||||
def addr_alias_str = '';
|
||||
if(d.addr_alias){
|
||||
addr_alias_str = d.addr_alias.join(" ")
|
||||
}
|
||||
rets << [d._id.toString(), d.city, d.region, d.name, d.address, addr_alias_str,d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
|
||||
}
|
||||
allrets.addAll(rets)
|
||||
if(allrets.size()>50000){
|
||||
ExcelUtils.write("D:\\数据\\小区\\小区精确地址补充"+index+".xlsx", allrets, "补充地址")
|
||||
allrets.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addressSelect(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-all-2.xlsx",{d->
|
||||
if(d['G']){
|
||||
d['O'] = AddressUtil.formatAddress(d['B'], d['G'])
|
||||
}
|
||||
if(d['I']){
|
||||
d['P'] = AddressUtil.formatAddress(d['B'], d['I'])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addrUpdateToDB(){
|
||||
def updateDist = 0
|
||||
def addAddrCount = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-入库-2.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def oldAddrAliasCount = 0
|
||||
if(dist.addr_alias){
|
||||
oldAddrAliasCount = dist.addr_alias.size()
|
||||
}
|
||||
def new_addr_alias = null
|
||||
if(d['K'].contains("新增百度地址") || d['K'].contains("双地图新增")){
|
||||
new_addr_alias = getNewAddressAlias(d['L'], dist.address, dist.addr_alias)
|
||||
}
|
||||
if(d['K'].contains("新增高德地址") || d['K'].contains("双地图新增")){
|
||||
new_addr_alias = getNewAddressAlias(d['M'], dist.address, dist.addr_alias)
|
||||
}
|
||||
if(new_addr_alias){
|
||||
updateDist++
|
||||
println updateDist
|
||||
addAddrCount = addAddrCount + new_addr_alias.size()-oldAddrAliasCount
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("addr_alias", new_addr_alias)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
println "修改小区数:"+updateDist
|
||||
println "新增地址数:"+addAddrCount
|
||||
}
|
||||
|
||||
def getNewAddressAlias(new_addr, old_addr, dist_addr_alias){
|
||||
def arrs = new_addr.split(";")
|
||||
def addr_alias = []
|
||||
if(dist_addr_alias){
|
||||
addr_alias = dist_addr_alias
|
||||
}
|
||||
for(def address : arrs){
|
||||
if(old_addr!=address && !addr_alias.contains(address)){
|
||||
addr_alias << address
|
||||
}
|
||||
}
|
||||
return addr_alias
|
||||
}
|
||||
|
||||
def updateDistAddr(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", d['J'])
|
||||
def addr_alias = dist.addr_alias
|
||||
if(!dist.addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
if(d['K'] && !addr_alias.contains(d['K'])){
|
||||
addr_alias << d['K']
|
||||
}
|
||||
|
||||
if(addr_alias){
|
||||
udata.put("addr_alias", addr_alias)
|
||||
}
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
d['L'] = "已更新"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def noDetailRoadInfoDistricts(districts){
|
||||
def rets = []
|
||||
for(def d : districts){
|
||||
if(d.address){
|
||||
def flag = DataHandleUtils.isStandardRoad(d.address)
|
||||
if(!flag){
|
||||
rets << d
|
||||
}
|
||||
}else{
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def haveRoadInfoDistricts(districts){
|
||||
def rets = []
|
||||
for(def d : districts){
|
||||
if(d.address){
|
||||
def flag = DataHandleUtils.isStandardRoad(d.address)
|
||||
if(flag){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = baiduProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.baidu_address = retMap.new_addr
|
||||
d.baidu_old_addr = retMap.old_addr
|
||||
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
def baiduProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
|
||||
for(def name : alias){
|
||||
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
|
||||
if(temp_address){
|
||||
retMap.old_addr = temp_address
|
||||
}
|
||||
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
|
||||
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
|
||||
retMap.new_addr = nineAddrSplit(fullCityName, temp_address)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def nineAddrSplit(city, addr){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
|
||||
def address = ""
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
|
||||
address = jsonObject.get("street")
|
||||
}
|
||||
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
|
||||
address = address + jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
return address
|
||||
}
|
||||
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
|
||||
def dbcursor = BAIDU_POI_COLL.find(q)
|
||||
def address = null
|
||||
for(def d : dbcursor){
|
||||
address = d.addr
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
|
||||
def gaodeMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = gaodeProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.gaode_addr = retMap.new_addr
|
||||
d.gaode_old_addr = retMap.old_addr
|
||||
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
|
||||
def gaodeProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
def address = null
|
||||
for(def name : alias){
|
||||
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
|
||||
if(temp_addr){
|
||||
retMap.old_addr = temp_addr
|
||||
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
|
||||
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
|
||||
retMap.new_addr = nineAddrSplit(fullCityName, temp_addr)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def findGaoDeDatas2(city, region, name){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
|
||||
def body = """
|
||||
{
|
||||
"_source": {
|
||||
"includes": ["cat", "address", "id","name"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"city" : "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"district" : "${region}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"name": {"query": "${name}","operator": "and"}
|
||||
}
|
||||
}
|
||||
// ,{
|
||||
// "match": {
|
||||
// "small_type": "住宅"
|
||||
// }
|
||||
// }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
|
||||
def address = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
def d = hits[0]
|
||||
if(d._source.cat.contains("住宅")){
|
||||
def source = d._source
|
||||
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
|
||||
if(newName == name){
|
||||
address = source.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return address
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -57,7 +57,7 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
//
|
||||
// }
|
||||
|
||||
checkRangeCount()
|
||||
// checkRangeCount()
|
||||
// synch100()
|
||||
}
|
||||
|
||||
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class AddrAlaisProcess {
|
||||
|
||||
static main(args) {
|
||||
def task = new AddrAlaisProcess()
|
||||
// task.run()
|
||||
task.nineApi()
|
||||
}
|
||||
|
||||
def nineApi(){
|
||||
def fileName = "/home/goojia/gj/district/addr_alias.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-测试.xlsx"
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
def addrMap = nineAddrSplit(d['B'], d['E'])
|
||||
def addrList = []
|
||||
if(addrMap && addrMap.address){
|
||||
d['G'] = addrMap.city
|
||||
d['H'] = addrMap.address
|
||||
addrList << addrMap.address
|
||||
}
|
||||
if(d['F']){
|
||||
def addrs = d['F'].split(",")
|
||||
def addr_alias = []
|
||||
for(def addr : addrs){
|
||||
def map = nineAddrSplit(d['B'], addr)
|
||||
if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
|
||||
if(!addrList.contains(map.address)){
|
||||
addrList << map.address
|
||||
addr_alias << map.address
|
||||
}
|
||||
}
|
||||
}
|
||||
d['I'] = addr_alias.join(",")
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def nineAddrSplit(city, addr){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
|
||||
def address = ""
|
||||
def map = [:]
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
|
||||
map.province = jsonObject.get("province")
|
||||
}else{
|
||||
map.province = ""
|
||||
}
|
||||
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
|
||||
map.city = jsonObject.get("city")
|
||||
}else{
|
||||
map.city = ""
|
||||
}
|
||||
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
|
||||
map.district = jsonObject.get("district")
|
||||
}else{
|
||||
map.district = ""
|
||||
}
|
||||
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
|
||||
address = jsonObject.get("street")
|
||||
}
|
||||
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
|
||||
address = address + jsonObject.get("streetNumber")
|
||||
}
|
||||
map.address = address
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rows = []
|
||||
citys.each{c->
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","address", "addr_alias"])
|
||||
println c +" "+districts.size()
|
||||
def num = 0
|
||||
for(def d : districts){
|
||||
num++
|
||||
def row = [num, d.city, d.region, d.name, d.address]
|
||||
if(d.addr_alias){
|
||||
row << d.addr_alias.join(",")
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区")
|
||||
FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.bat", null, "utf-8", false, "#","");
|
||||
}
|
||||
}
|
||||
+11
-11
@@ -35,22 +35,22 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// //查找第三方匹配没有匹配上的小区
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def list = []
|
||||
// list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
|
||||
// for(def c : citys){
|
||||
// def noMatchDatas = findNoMatchResult(c)
|
||||
// println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||
// def rets = checkDatas(noMatchDatas.keySet())
|
||||
// list.addAll(rets)
|
||||
// }
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
|
||||
for(def c : citys){
|
||||
def noMatchDatas = findNoMatchResult(c)
|
||||
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||
def rets = checkDatas(noMatchDatas.keySet())
|
||||
list.addAll(rets)
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\Êý¾Ý\\add_dist_info_1.xlsx"
|
||||
}
|
||||
// ExcelUtils.write(fileName, list, "待入库数据")
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
|
||||
|
||||
check(fileName)
|
||||
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class BusinessQuarterProcess {
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020)
|
||||
|
||||
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new BusinessQuarterProcess()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
// synchBusinessData()
|
||||
// updateBusinessQuarter()
|
||||
// addRegion()
|
||||
synchBusinessToEs()
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
def list = CommonDatautils.queryRangeByCat("区域")
|
||||
println list.size()
|
||||
def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2")))
|
||||
for(def d : dbcursor){
|
||||
if(d.district){
|
||||
continue
|
||||
}
|
||||
for(def regionShape : list){
|
||||
if(d.short_city!=regionShape.city_short){
|
||||
continue
|
||||
}
|
||||
def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates))
|
||||
if(flag){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district", regionShape.mzb_district_name)
|
||||
udata.put("short_district", regionShape.district_short)
|
||||
bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def synchBusinessData(){
|
||||
def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList()
|
||||
synchMongoProcess(datas)
|
||||
|
||||
}
|
||||
|
||||
def synchMongoProcess(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
createData(d)
|
||||
println d.city +" "+totalCount.get() +"/"+datas.size()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def createData(d){
|
||||
def row = new BasicDBObject()
|
||||
row.put("_id", d._id)
|
||||
row.put("short_province", d.province)
|
||||
def short_city = RegionData.formatCity(d.city)
|
||||
row.put("short_city", short_city)
|
||||
row.put("name", d.name)
|
||||
row.put("bd_location", ["lon":DataHandleUtils.getDoubleRound(6,d.longitude), "lat":DataHandleUtils.getDoubleRound(6,d.latitude)])
|
||||
|
||||
def data = fullCityMap.get(short_city)
|
||||
if(data){
|
||||
row.put("province", data.province)
|
||||
row.put("city", data.city)
|
||||
}else{
|
||||
println d.city + " 没找到简称"
|
||||
}
|
||||
|
||||
|
||||
def gdPoints = FangjiaClientUtils.convert(d.longitude+","+d.latitude, 3)
|
||||
if(gdPoints){
|
||||
row.put("location", ["lon":DataHandleUtils.getDoubleRound(6,gdPoints[0]), "lat":DataHandleUtils.getDoubleRound(6,gdPoints[1])])
|
||||
}
|
||||
def border = d.border
|
||||
def bd_shapes = []
|
||||
def gd_shapes = []
|
||||
if(border){
|
||||
def arrs = border.split(";")
|
||||
if(border.contains("|")){
|
||||
println d._id.toString()+" 可能含多个围栏"
|
||||
}
|
||||
|
||||
for(def i=0;i<arrs.size();i++){
|
||||
def str = arrs[i]
|
||||
def arr = str.split(",")
|
||||
def bd_lng = DataHandleUtils.getDoubleRound(6, arr[0].toString().toDouble())
|
||||
def bd_lat = DataHandleUtils.getDoubleRound(6, arr[1].toString().toDouble())
|
||||
if(!bd_shapes.contains([bd_lng, bd_lat]) ){
|
||||
bd_shapes << [bd_lng, bd_lat]
|
||||
}
|
||||
if(bd_shapes.contains([bd_lng, bd_lat]) && i+1==arrs.size() && bd_shapes[0]==[bd_lng, bd_lat]){
|
||||
bd_shapes << [bd_lng, bd_lat]
|
||||
}
|
||||
def tempGdPoints = FangjiaClientUtils.convert(bd_lng+","+bd_lat, 3)
|
||||
if(tempGdPoints){
|
||||
def gdlist = [DataHandleUtils.getDoubleRound(6,tempGdPoints[0]), DataHandleUtils.getDoubleRound(6,tempGdPoints[1])]
|
||||
if(!gd_shapes.contains(gdlist)) {
|
||||
gd_shapes << gdlist
|
||||
}
|
||||
if(gd_shapes.contains(gdlist) && i+1==arrs.size() && gd_shapes[0]==gdlist){
|
||||
gd_shapes << gdlist
|
||||
}
|
||||
}
|
||||
}
|
||||
if(bd_shapes[0] != bd_shapes[bd_shapes.size()-1]){
|
||||
bd_shapes << bd_shapes[0]
|
||||
gd_shapes << gd_shapes[0]
|
||||
}
|
||||
}
|
||||
def gdMap = [:]
|
||||
gdMap.put("type" , "polygon")
|
||||
gdMap.put("coordinates", [gd_shapes])
|
||||
row.put("shape", gdMap)
|
||||
|
||||
def bdMap = [:]
|
||||
bdMap.put("type" , "polygon")
|
||||
bdMap.put("coordinates", [bd_shapes])
|
||||
row.put("bd_shape", bdMap)
|
||||
row.put("c_date", new Date())
|
||||
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
|
||||
}
|
||||
def updateBusinessQuarter(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
if(["上海","北京"].contains(c)){
|
||||
continue
|
||||
}
|
||||
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
|
||||
businessProcess(c, districts, cityBusinessList)
|
||||
}
|
||||
}
|
||||
|
||||
def businessProcess(c,districts, cityBusinessList){
|
||||
def updateIndex = 0
|
||||
for(def d : districts){
|
||||
if(!d.business_quarter){
|
||||
def businessQuarter = findBusinessByRange(d, cityBusinessList)
|
||||
if(businessQuarter){
|
||||
updateIndex ++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("business_quarter", businessQuarter)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println c +" "+ districts.size() +" "+updateIndex
|
||||
}
|
||||
def findBusinessByRange(d, cityBusinessList){
|
||||
def name= null
|
||||
for(def shapeObj : cityBusinessList){
|
||||
if(d.lng){
|
||||
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(shapeObj.shape.coordinates))
|
||||
if(flag){
|
||||
name = shapeObj.name
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
def synchBusinessToEs(){
|
||||
def dbcursor = bk_business_coll.find(new BasicDBObject())
|
||||
def rows = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
if(index<5000){
|
||||
continue
|
||||
}
|
||||
// rows << d
|
||||
// if(rows.size()>=100){
|
||||
// updateByEs(rows)
|
||||
// rows.clear()
|
||||
// }
|
||||
updateByEs(d, index)
|
||||
}
|
||||
}
|
||||
|
||||
def updateByEs(row, index){
|
||||
BulkRequest request = new BulkRequest();
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
int processedCount = 0;
|
||||
|
||||
def id = row.get("_id").toString()
|
||||
def doc = [:]
|
||||
doc.put("province",row.province)
|
||||
doc.put("short_province",row.short_province)
|
||||
doc.put("city",row.city)
|
||||
doc.put("short_city",row.short_city)
|
||||
doc.put("district",row.district)
|
||||
doc.put("short_district",row.short_district)
|
||||
doc.put("name", row.name)
|
||||
doc.put("location", row.location)
|
||||
doc.put("bd_location", row.bd_location)
|
||||
doc.put("shape", row.shape)
|
||||
doc.put("bd_shape", row.bd_shape)
|
||||
|
||||
|
||||
request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON));
|
||||
if (request.numberOfActions() >= 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
// println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+17
-13
@@ -72,7 +72,7 @@ class DevProcess {
|
||||
// println DateUtils.add("2024-7-24", 7);
|
||||
|
||||
// task.addKfc()
|
||||
// task.devToPark() //开发区下辖园区
|
||||
// task.devToPark() //开发区下辖园
|
||||
// task.checkTown()
|
||||
|
||||
// task.repeatSave()
|
||||
@@ -84,6 +84,7 @@ class DevProcess {
|
||||
// task.addQiDevResult() //获取齐梁数据结果
|
||||
|
||||
// task.updateDevShape()//修改开发区结构
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -439,30 +440,33 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
|
||||
if(!d['B'] && d['A']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
|
||||
if(!d['E'] && d['D']){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d['A'])
|
||||
// q.put("city",d['G'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['A']]))
|
||||
def name = d['A'].replace('经济技术开发区','经济开发区')
|
||||
// q.put("name", d['D'])
|
||||
q.put("city",d['C'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['D']]))
|
||||
def name = d['D'] //.replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['I'] = "库中有开发区-相似2"
|
||||
d['G'] = "相似"
|
||||
// d['K'] = obj.name
|
||||
// d['C'] = obj._id.toString()
|
||||
d['B'] = obj.name
|
||||
// if(obj.name == d['A']){
|
||||
// d['E'] = "相同"
|
||||
d['E'] = obj.name
|
||||
// if(obj.name == d['D']){
|
||||
// d['G'] = "相同"
|
||||
// }else{
|
||||
// d['E'] = "包含"
|
||||
// d['G'] = "包含"
|
||||
// }
|
||||
if(obj.shape){
|
||||
d['F'] = "是"
|
||||
}
|
||||
// d['E'] = "别名"
|
||||
// d['F'] = obj.name
|
||||
}
|
||||
}
|
||||
},1);
|
||||
},0);
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.script.gj.ProxyHttpUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
class DevQlCheck {
|
||||
|
||||
//http://146.56.217.44:8088/uams/index
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevQlCheck()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def devName = "福建龙海经济开发区"
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",devName))
|
||||
def pois = readCSV("C:\\Users\\gaojuan\\Downloads\\漳州市POI.csv")
|
||||
def devPois = []
|
||||
devPois << ["省", "城市", "区域", "名称", "地址", "类型", "经度", "纬度", "齐梁开发区"]
|
||||
for(def poi : pois){
|
||||
def flag = InsidePolygon.isInArea(poi.lng.toString().toDouble(), poi.lat.toString().toDouble(), InsidePolygon.createPoint4(obj.shape.coordinates))
|
||||
if(flag){
|
||||
devPois << [poi.province, poi.city, poi.district, poi.name, poi.address, poi.cat, poi.lng, poi.lat]
|
||||
}
|
||||
}
|
||||
println "共:"+devPois.size()+" 个"
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx", devPois, "齐梁开发区")
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx",{d->
|
||||
def address = d['B']+d['C']+d['D']
|
||||
// if(d['E']){
|
||||
// address = address + d['E']
|
||||
// }
|
||||
Thread.sleep(3000)
|
||||
def s = getQlResult(address)
|
||||
d['I'] = s
|
||||
index ++
|
||||
println index+" "+ s
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
|
||||
def getQlResult(address){
|
||||
def params = [
|
||||
'address': address,
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
try{
|
||||
def str = ProxyHttpUtils.getProxyResult(targetUrl, params);
|
||||
println address +" "+ str
|
||||
JSONObject result = JSONObject.fromObject(str)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println address +" "+e.getMessage()
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("province", arrs[0])
|
||||
o.put("city", arrs[1])
|
||||
o.put("district", arrs[2])
|
||||
o.put("name", arrs[3])
|
||||
o.put("township", arrs[4])
|
||||
o.put("address", arrs[5])
|
||||
o.put("cat", arrs[6])
|
||||
o.put("lng", arrs[7])
|
||||
o.put("lat", arrs[8])
|
||||
o.put("data_source", arrs[9])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println filename+" 文件异常"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -32,7 +32,7 @@ class DevUpdateTestBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
task.addDevAddress(path)
|
||||
//
|
||||
@@ -167,7 +167,7 @@ class DevUpdateTestBatch {
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+140
-39
@@ -1,12 +1,14 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -28,14 +30,24 @@ class GXDdistrictMatch {
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
task.addOldMatch2()
|
||||
// task.addOldMatch2()
|
||||
// task.runDb()
|
||||
// task.hainanMatch()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
// task.runMatch()
|
||||
|
||||
// task.checkDist()
|
||||
|
||||
task.addPrice()
|
||||
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
@@ -94,6 +106,46 @@ class GXDdistrictMatch {
|
||||
|
||||
}
|
||||
|
||||
def hainanMatch(){
|
||||
def fileName = "D:\\数据\\小区\\海南未匹配地址-1.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['D'] && d['F'] ){//
|
||||
// def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['I'])
|
||||
// if(jsonObject){
|
||||
//// if(jsonObject.get("district") && jsonObject.get("district")!='null'){
|
||||
//// d['AO'] = jsonObject.get("district")
|
||||
//// }
|
||||
// if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
// d['AP'] = jsonObject.get("province")
|
||||
// }
|
||||
// if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
// d['AI'] = jsonObject.get("city")
|
||||
// }
|
||||
// }
|
||||
def formatCity = RegionData.formatCity(d['F'])
|
||||
def fullCityObj = fullCityMap.get(formatCity)
|
||||
if(fullCityObj){
|
||||
// def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
|
||||
|
||||
def obj = findDatasByCityAndName(formatCity, d['D'].replaceAll('[·\\(\\)()]|·', ''))
|
||||
if(obj){
|
||||
d['U'] = "楼盘抓取库"
|
||||
d['V'] = obj.city
|
||||
d['W'] = obj.region
|
||||
d['X'] = obj.name
|
||||
d['Y'] = obj.property_type
|
||||
d['Z'] = obj.address
|
||||
def loc = obj.location
|
||||
if(loc){
|
||||
d['AA'] = loc.x
|
||||
d['AB'] = loc.y
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def runDb(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
@@ -238,43 +290,41 @@ class GXDdistrictMatch {
|
||||
|
||||
def saveToDB(){
|
||||
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
|
||||
def fileName = "D:\\数据\\小区\\海南新增小区.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['A']){
|
||||
if(d['A'] && d['C']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def alias = []
|
||||
if(d['G'] && name!=d['G']){
|
||||
alias << d['G']
|
||||
}
|
||||
// def alias = []
|
||||
// if(d['G'] && name!=d['G']){
|
||||
// alias << d['G']
|
||||
// }
|
||||
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
// alias << d['AI']
|
||||
// }
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
// if(alias){
|
||||
// udata.put("alias", alias)
|
||||
// }
|
||||
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['E'])
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("status", 0)
|
||||
udata.put("name_cat", 0)
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
// udata.put("username", "高娟")
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
// if(d['AH']){
|
||||
// type = d['AH']
|
||||
// }else{
|
||||
// type = d['Y']
|
||||
// }
|
||||
if(d['D']){
|
||||
type = d['D']
|
||||
}
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
@@ -282,26 +332,50 @@ class GXDdistrictMatch {
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
udata.put("estate_type2", estateType2)
|
||||
|
||||
if(d['F']){
|
||||
Long lng = d['F']*10000000000
|
||||
udata.put("lng", lng)
|
||||
}
|
||||
if(d['E']){
|
||||
Long lat = d['E'] * 10000000000
|
||||
Long lat = d['G'] * 10000000000
|
||||
udata.put("lat", lat)
|
||||
}
|
||||
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
def opt = "add"
|
||||
if(s && s=='成功'){
|
||||
saveOptLog(udata.city, udata.region, udata.name, opt)
|
||||
udata.put("lng", lng)
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(d['F'].toString().toDouble(), d['G'].toString().toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
if(lng2){
|
||||
Long lng2_new = lng2 *10000000000L
|
||||
Long lat2_new = lat2 *10000000000L
|
||||
udata.put("lng2", lng2_new)
|
||||
udata.put("lat2", lat2_new)
|
||||
}
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map",markMap)
|
||||
// udata.put("is_new", 0)
|
||||
DbUtils.seaweed.save(udata)
|
||||
saveOptLog(udata.city, udata.region, udata.name, "add")
|
||||
// def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println name
|
||||
// def opt = "add"
|
||||
// if(s && s=='成功'){
|
||||
// saveOptLog(udata.city, udata.region, udata.name, opt)
|
||||
// }
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def checkDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\海南未匹配地址-1.xlsx", { d ->
|
||||
if(d['V'] && d['W'] && d['X']){
|
||||
matching(d['V'], d['W'], d, d['X'], null, null)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(city, region, d){
|
||||
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
|
||||
@@ -391,7 +465,7 @@ class GXDdistrictMatch {
|
||||
if(rets && rets.size()>=1){
|
||||
d = setRowValue(rets[0], d, "名称在地址中"+str)
|
||||
}else{
|
||||
if (d['D']) {
|
||||
if (d['D'] && alias) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
@@ -449,14 +523,14 @@ class GXDdistrictMatch {
|
||||
|
||||
def setRowValue(dbo, d, remark){
|
||||
if (dbo) {
|
||||
d['V'] = dbo.city
|
||||
d['W'] = dbo.region
|
||||
d['X'] = dbo.name
|
||||
d['Y'] = dbo.estate_type2
|
||||
d['Z'] = dbo.status
|
||||
d['AA'] = dbo.name_cat
|
||||
d['U'] = remark
|
||||
d['AH'] = dbo.address
|
||||
// d['V'] = dbo.city
|
||||
// d['W'] = dbo.region
|
||||
// d['X'] = dbo.name
|
||||
// d['Y'] = dbo.estate_type2
|
||||
d['AK'] = dbo.status
|
||||
// d['AA'] = dbo.name_cat
|
||||
d['AJ'] = remark
|
||||
// d['AH'] = dbo.address
|
||||
}
|
||||
return d;
|
||||
}
|
||||
@@ -510,6 +584,33 @@ class GXDdistrictMatch {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def findDatasByCityAndName(def city, def name){
|
||||
def sql = "select * from community where city='${city}' and name='${name}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
println d.property_type
|
||||
})
|
||||
|
||||
if(rets){
|
||||
def obj = null
|
||||
for(def r : rets){
|
||||
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
|
||||
obj = r
|
||||
break
|
||||
}
|
||||
}
|
||||
if(obj){
|
||||
return obj;
|
||||
}else{
|
||||
return rets[0];
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, type){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
|
||||
+185
-3
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -39,12 +46,13 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
// run()
|
||||
oldPoiMagre()
|
||||
}
|
||||
|
||||
def run(){
|
||||
if (CommonDatautils.isWindow()) {
|
||||
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
|
||||
TAGS = getAllPoiType("D:\\数据\\数据建设\\Poi\\POI分类白名单.xlsx")
|
||||
}else{
|
||||
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
|
||||
}
|
||||
@@ -145,6 +153,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
saveObj.data_id = map.map_id
|
||||
saveObj.data_source = "¸ßµÂµØÍ¼"
|
||||
saveObj.data_source_year = "2024-11"
|
||||
saveObj.parent_id = map.parent_id
|
||||
|
||||
if(saveObj.cat && TAGS.contains(saveObj.cat)){
|
||||
saveObj.status = 0
|
||||
@@ -314,7 +323,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
rows.each{map->
|
||||
// def map = rows.get(key)
|
||||
|
||||
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
|
||||
|
||||
// ÅúÁ¿Ìá½»Êý¾Ý
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
@@ -346,4 +355,177 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def oldPoiMagre(){
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
def index = 0
|
||||
for(def cityObj : cityList){
|
||||
index ++
|
||||
if(index <291){
|
||||
continue
|
||||
}
|
||||
def fileName = "/home/goojia/gj/poi/old_poi/merge_poi/${cityObj.city}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
|
||||
}
|
||||
println fileName
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
def results = readCSV(fileName)
|
||||
println "运行城市:"+cityObj.province+" " +cityObj.city+" "+results.size()
|
||||
if(results){
|
||||
def megerMap = poiRepeat(cityObj.city, results)
|
||||
println "待保存:" + cityObj.city +" "+megerMap.size()
|
||||
save(megerMap.values())
|
||||
}
|
||||
}else{
|
||||
println fileName +" 不存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def poiRepeat(city, oldPois){
|
||||
def rets = new ConcurrentHashMap()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(oldPois.size())
|
||||
def townMap = CommonDatautils.queryTownByCity(city)
|
||||
def esMap = findEsPoiByCity(city)
|
||||
|
||||
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
for(def i=0;i<oldPois.size();i++){
|
||||
final def p = oldPois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def nameMap = esMap.get(p.district)
|
||||
def nameKeys = nameMap.keySet()
|
||||
def key = p.name.replace("-","").replace("·","")
|
||||
if(!nameKeys.contains(key)){
|
||||
def simlarObj = null
|
||||
if(esMap){
|
||||
simlarObj = findSimlarPoi(p, esMap)
|
||||
}
|
||||
if(!simlarObj){
|
||||
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
|
||||
if(!rets.containsKey(newKey)){
|
||||
p.data_source_year = "2023-08"
|
||||
rets.put(newKey, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findSimlarPoi(ret, esMap){
|
||||
def simlarObj = null
|
||||
def region = ""
|
||||
if(ret.district){
|
||||
region = ret.district
|
||||
}
|
||||
def regionMap = esMap.get(region)
|
||||
if(regionMap){
|
||||
def keys = regionMap.keySet()
|
||||
|
||||
for(def key : keys){
|
||||
def esValue = regionMap.get(key)
|
||||
if(esValue.township && ret.township){
|
||||
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
||||
if(ret.location && ret.location.lon){
|
||||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||
if(distance <100){
|
||||
simlarObj = esValue
|
||||
println esValue.name +" "+ret.name+" "+distance
|
||||
break;
|
||||
}
|
||||
}else{
|
||||
println ret.data_id +" 无经纬度信息"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "无数据"
|
||||
}
|
||||
|
||||
return simlarObj
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def map = new HashMap()
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "map_poi_2024", queryBuilder, { String id, Map d->
|
||||
def district = "嘉峪关市"
|
||||
if(d.district){
|
||||
district = d.district
|
||||
}
|
||||
def regionMap = map.get(district)
|
||||
if(!regionMap){
|
||||
regionMap = new HashMap()
|
||||
}
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
regionMap.put(d.name.replace("-","").replace("·",""), o)
|
||||
// map.put(district, regionMap)
|
||||
map.put(district, regionMap)
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("province",arrs[0])
|
||||
o.put("city" , arrs[1])
|
||||
o.put("district",arrs[2])
|
||||
o.put("name",arrs[3])
|
||||
o.put("township",arrs[4])
|
||||
o.put("address",arrs[5])
|
||||
o.put("cat",arrs[6])
|
||||
o.put("type",arrs[7])
|
||||
o.put("secondary_type",arrs[8])
|
||||
o.put("small_type" ,arrs[9])
|
||||
if(arrs[10]){
|
||||
o.put("data_id",arrs[10])
|
||||
}else{
|
||||
o.put("data_id",arrs[11])
|
||||
}
|
||||
|
||||
o.put("spell_name",arrs[12])
|
||||
def location = [:]
|
||||
if(arrs[13]){
|
||||
location.lon = arrs[13].toString().toDouble()
|
||||
location.lat = arrs[14].toString().toDouble()
|
||||
}
|
||||
o.put("location",location)
|
||||
o.put("data_source",arrs[15])
|
||||
o.put("status",arrs[16])
|
||||
o.put("version",arrs[17])
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class PoiNineSplitBatch{
|
||||
|
||||
static main(args) {
|
||||
public static void main(args) {
|
||||
def task = new PoiNineSplitBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
-6
@@ -52,9 +52,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 206){
|
||||
continue
|
||||
}
|
||||
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
|
||||
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||
matchByCity(c, noMatchDatas)
|
||||
@@ -123,9 +120,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 70){
|
||||
continue
|
||||
}
|
||||
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
|
||||
def addCount = addDatasToDB(crawlDatas)
|
||||
println c + " ÐÂÔö£º"+ addCount
|
||||
|
||||
+371
-30
@@ -3,15 +3,29 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
@@ -22,6 +36,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def updateList = []
|
||||
def addAliasList = []
|
||||
@@ -38,7 +53,9 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def TownStandard(){
|
||||
// townStandardByCat("office")
|
||||
townStandardByPoi()
|
||||
// townStandardByPoi()
|
||||
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
|
||||
// updateTownData()
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
|
||||
@@ -69,11 +86,15 @@ class TownStandardBatch extends BaseBatch{
|
||||
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||
|
||||
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
|
||||
//浙江,江苏,安徽
|
||||
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
|
||||
def runProvinces = ["福建省","浙江省","江苏省","安徽省","山东省","甘肃省"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
|
||||
index++
|
||||
def data = fullCityMap.get(city)
|
||||
// if(runProvinces.contains(data.province)){
|
||||
// continue
|
||||
// }
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
@@ -95,11 +116,16 @@ class TownStandardBatch extends BaseBatch{
|
||||
def rets = readPoiCsv(fileName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
if(updateList.size()>50000){
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-${index}.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名-${index}.xlsx", addAliasList, "数据")
|
||||
updateList.clear()
|
||||
}
|
||||
}else{
|
||||
println fileName+" 文件没找到"
|
||||
}
|
||||
@@ -107,8 +133,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-最后.xlsx", updateList, "数据")
|
||||
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
@@ -168,7 +193,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
for(def d : rets){
|
||||
@@ -221,21 +246,21 @@ class TownStandardBatch extends BaseBatch{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":updateTown])
|
||||
results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
updateList << [d.id, d.province, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":""])
|
||||
// results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
updateList << [d.id, d.province, d.city, d.district, d.name, d.township, d.status, ""]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
|
||||
}
|
||||
}
|
||||
@@ -244,10 +269,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
// if(results.size()>0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
if(results.size()>0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println fullCityName +" 处理完成"
|
||||
}
|
||||
|
||||
@@ -322,18 +347,334 @@ class TownStandardBatch extends BaseBatch{
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
try{
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def readCSV2(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
def index = 0
|
||||
for(line in lines) {
|
||||
if(line && index>=1){
|
||||
String[] arrs = line.split(",")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("id", arrs[1])
|
||||
o.put("province", arrs[2])
|
||||
o.put("city", arrs[3])
|
||||
o.put("district", arrs[4])
|
||||
o.put("lng", arrs[5])
|
||||
o.put("lat", arrs[6])
|
||||
if(arrs.size()<=7){
|
||||
o.put("data_source","")
|
||||
}else{
|
||||
o.put("data_source", arrs[7])
|
||||
}
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
}
|
||||
index++
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def readCSV3(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
def index = 0
|
||||
for(line in lines) {
|
||||
if(line && index>=1){
|
||||
String[] arrs = line.split(",")
|
||||
if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("id", arrs[0])
|
||||
o.put("township", arrs[7])
|
||||
if(arrs.size()>=9 && arrs[8]){
|
||||
o.put("district", arrs[8])
|
||||
}
|
||||
if(arrs.size()>=10 && arrs[9]){
|
||||
o.put("way", arrs[9])
|
||||
}
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
index++
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addTownByTownShape(){
|
||||
def i = 0
|
||||
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
|
||||
while(i<50){
|
||||
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()) {
|
||||
def rets = readCSV2(fileName)
|
||||
println i +" "+ rets.size()
|
||||
def rows = addTownshipBatch(rets)
|
||||
if (CommonDatautils.isWindow()) {
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}else{
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}
|
||||
println fileName +" 运行完成"
|
||||
}else{
|
||||
println fileName +" 不存在"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
def addTownshipBatch(def rets){
|
||||
def results = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def r = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def rMap = regionFullNameMap.get(r.city)
|
||||
if (rMap) {
|
||||
def townMap = [:]//CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
|
||||
// def set = rMap.keySet()
|
||||
println totalCount.get() +" "+r.city + " "
|
||||
// if (set.size() >= 1) {
|
||||
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
|
||||
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
|
||||
// }
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return results
|
||||
}
|
||||
|
||||
def addTown(saveObj, townMap, location) {
|
||||
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
|
||||
// def towns = townMap.get(saveObj.district)
|
||||
// if (towns) {
|
||||
// for (def town : towns) {
|
||||
// if (town.shape && town.shape.coordinates) {
|
||||
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
// if (flag) {
|
||||
// saveObj.township = town.street
|
||||
// saveObj.town_source = "同区域"
|
||||
// saveObj.strand_town = "是"
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def townNames = townMap.keySet()
|
||||
// for(def townName : townNames){
|
||||
// towns = townMap.get(townName)
|
||||
// for (def town : towns) {
|
||||
// if (town.shape && town.shape.coordinates) {
|
||||
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
// if (flag) {
|
||||
// saveObj.township = town.street
|
||||
// saveObj.new_district = town.mzb_district_name
|
||||
// saveObj.town_source = "忽略区域"
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(saveObj.township){
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//通过调取地图api接口获取乡镇街道
|
||||
if(!saveObj.township){
|
||||
def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
saveObj.township = gaodeMap.township
|
||||
saveObj.new_district = gaodeMap.district
|
||||
saveObj.town_source = "高德API"
|
||||
saveObj.strand_town = ""
|
||||
// if(townMap.size()>=1){
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
// //通过调取地图api接口获取乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
|
||||
// if(gaodeMap && gaodeMap.addressComponent) {
|
||||
// def addressComponent = gaodeMap.addressComponent
|
||||
// if(addressComponent){
|
||||
// if(addressComponent.town){
|
||||
// saveObj.township = addressComponent.town
|
||||
// saveObj.new_district = addressComponent.district
|
||||
// saveObj.town_source = "百度API"
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def isStandTown(townMap, district, town){
|
||||
def towns = townMap.get(district)
|
||||
def flag = "非"
|
||||
if(towns){
|
||||
for(def t : towns){
|
||||
if(t.street == town){
|
||||
flag = "是"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
def jsonObject = null
|
||||
// while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
city = addressComponent.city
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(jsonObject.status==0 || count>3){
|
||||
// stop = true
|
||||
// println "高德调用超3次"
|
||||
// }else{
|
||||
// count ++
|
||||
// Thread.sleep(3000)
|
||||
// print "."
|
||||
// }
|
||||
// }
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateTownData(){
|
||||
def i = 0
|
||||
while(i<50) {
|
||||
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv"
|
||||
}
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
def rets = readCSV3(fileName)
|
||||
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
|
||||
// if(rets){
|
||||
// updateToEs(rets)
|
||||
// }
|
||||
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.containsKey("way")){
|
||||
map.remove("way")
|
||||
}
|
||||
request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
-4
@@ -38,16 +38,16 @@ class TownshipStrandUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
// task.process()
|
||||
// task.updateTown()
|
||||
task.process()
|
||||
task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
task.updateDiffRegionTown()
|
||||
// task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
// task.updateDevMapping()
|
||||
println new Date().getTime()
|
||||
// println new Date().getTime()
|
||||
// task.updateSpecCityTown()
|
||||
}
|
||||
|
||||
|
||||
+144
-21
@@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
||||
|
||||
//
|
||||
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
||||
def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房",
|
||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
|
||||
|
||||
@@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
|
||||
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
|
||||
final static def EMAILS = [
|
||||
"gaojuan@fangjia.com"
|
||||
@@ -68,6 +71,38 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictFieldAddBatch(), args)
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
|
||||
//// if(dist){
|
||||
//// d['L'] = dist.loudong_count
|
||||
//// d['M'] = dist.household_count
|
||||
//// d['Q'] = dist.estate_type2
|
||||
//// }
|
||||
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
|
||||
// if(!list && dist){
|
||||
// def alias = dist.alias
|
||||
// for(def name : alias){
|
||||
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
|
||||
// if(list){
|
||||
// d['Z'] = "别名"
|
||||
// }
|
||||
// }
|
||||
// }else{
|
||||
// d['Z'] = "名称"
|
||||
// }
|
||||
// if(list){
|
||||
// def obj = list[0]
|
||||
// if(obj.loudong_cnt){
|
||||
// d['W'] = obj.loudong_cnt
|
||||
// }
|
||||
//
|
||||
// if(obj.households){
|
||||
// d['X'] = obj.households
|
||||
// }
|
||||
// d['Y'] = obj.url
|
||||
// }
|
||||
//
|
||||
// })
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -83,18 +118,13 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
headers << fieldsNameMap.get(it)
|
||||
}
|
||||
|
||||
headers << "抓取量"
|
||||
headers << "匹配量"
|
||||
headers << "0.9分以上量"
|
||||
headers << "匹配率"
|
||||
headers << "0.9以上匹配率"
|
||||
// analysRets << headers
|
||||
|
||||
for(def c : citys){
|
||||
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||
if(totalCount){
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
||||
// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
||||
analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表
|
||||
}
|
||||
}
|
||||
println analysRets.toString()
|
||||
@@ -107,8 +137,10 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findEsDatas(c, esWebSource)
|
||||
|
||||
def matchRets1 = matchApi(esRets)
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
|
||||
@@ -140,6 +172,65 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def addFieldFromMatchMapping(c, totalCount, analysRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findMatchSameDatas(c, esWebSource)
|
||||
println esWebSource +" 新老算法匹配相同量:"+esRets.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , esRets, esWebSource)
|
||||
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
|
||||
def analyRet = []
|
||||
analyRet << seaweedmapSource
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analysRets << analyRet
|
||||
}
|
||||
return analysRets;
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findMatchSameDatas(c, esWebSource){
|
||||
def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true))
|
||||
def siteDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||
def rows = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(siteDatas.size())
|
||||
for(def i=0;i<siteDatas.size();i++){
|
||||
final def d = siteDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findCommunities(d.url)
|
||||
if(obj){
|
||||
obj.match_id = d.match_id
|
||||
rows << obj
|
||||
}
|
||||
// println d.city +" "+totalCount.get() +"/"+siteDatas.size()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addFieldFromRonghe(c, totalCount, rh_analysRets){
|
||||
@@ -209,14 +300,10 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
fields << "city"
|
||||
fields << "estate_type2"
|
||||
def fieldValueMap = [:]
|
||||
def rets = []
|
||||
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
|
||||
for(def matchDist : matchRets){
|
||||
|
||||
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
|
||||
// continue
|
||||
// }
|
||||
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
println c+" "+matchDist.match_id+" 不存在"
|
||||
@@ -224,7 +311,6 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
def newValues = strandField(c, dist, matchDist)
|
||||
if(newValues){
|
||||
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
@@ -244,7 +330,9 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
if(newVal) {
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
@@ -312,12 +400,16 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
if(seaweedfield == "lng"){
|
||||
def locationMap = FiledStrand.processLngAndLat(esVal)
|
||||
if(locationMap){
|
||||
newValues.put("lat", locationMap.lat)
|
||||
newValues.put("lng", locationMap.lng)
|
||||
newValues.put("lat2", locationMap.lat2)
|
||||
newValues.put("lng2", locationMap.lng2)
|
||||
try{
|
||||
def locationMap = FiledStrand.processLngAndLat(esVal)
|
||||
if(locationMap){
|
||||
newValues.put("lat", locationMap.lat)
|
||||
newValues.put("lng", locationMap.lng)
|
||||
newValues.put("lat2", locationMap.lat2)
|
||||
newValues.put("lng2", locationMap.lng2)
|
||||
}
|
||||
}catch(Exception e){
|
||||
println esVal
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -361,6 +453,15 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
return list
|
||||
}
|
||||
|
||||
|
||||
static def findEsDatasByCityAndName(c, district, name, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' and region='${district}' and name='${name}'", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
@@ -621,4 +722,26 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
|
||||
}
|
||||
}
|
||||
|
||||
def findCommunities(url){
|
||||
if(!url){
|
||||
return null
|
||||
}
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||
def rets = []
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
}catch(Exception e){
|
||||
findCommunities(url)
|
||||
}
|
||||
def obj = null
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
obj = d
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictStatRentCleanBatch extends BaseBatch{
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictStatRentCleanBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def keys = CommonDatautils.getSimpleDistricts("district", c)
|
||||
running(keys, c)
|
||||
println c +" ÔËÐÐÍê³É"
|
||||
}
|
||||
}
|
||||
|
||||
def running(keys, c){
|
||||
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
|
||||
def fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("district_name", 1)
|
||||
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
|
||||
def delRets = []
|
||||
for(def d : dbcursor){
|
||||
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
|
||||
println d.city+"@"+d.region+"@"+d.district_name
|
||||
delRets << d._id
|
||||
}
|
||||
if(delRets.size()>=100){
|
||||
def delquery = new BasicDBObject()
|
||||
delquery.put("_id", new BasicDBObject('$in', delRets))
|
||||
RENT_DIST_STAT_COLL.remove(delquery)
|
||||
delRets.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+28
@@ -398,4 +398,32 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 经纬度转换
|
||||
* 1、高德转百度 2、高德转WGS84 3、百度转高德 4、WGS84转高德 5、百度转WGS84 6、WGS84转百度
|
||||
*/
|
||||
public static def convert(String locationstr, type){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
def points = []
|
||||
try {
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/convert?locations="+URLEncoder.encode(locationstr,"UTF-8")+"&coordsys=${type}"
|
||||
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
if(jsonObj && jsonObj.locations){
|
||||
def locations = jsonObj.locations
|
||||
for(def s : locations){
|
||||
def arrs = s.split(",")
|
||||
points = [arrs[0].toString().toDouble(), arrs[1].toString().toDouble()]
|
||||
}
|
||||
}
|
||||
return points
|
||||
} catch (Exception e) {
|
||||
println e.getMessage()
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 根据链家数据修改小区库中字段
|
||||
* 使用到中间映射表
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class UpdateDistrictField extends BaseBatch{
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
|
||||
|
||||
def sources = ["链家", "我爱我家", "安居客"]
|
||||
|
||||
def statMap = [:]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new UpdateDistrictField(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def sourceMap = [:]
|
||||
sources.each {source->
|
||||
def valMap = findThirdMatchDist(source)
|
||||
println source +" "+valMap.size()
|
||||
if(valMap){
|
||||
sourceMap.put(source, valMap)
|
||||
}
|
||||
}
|
||||
def margeResults = megerDatas(sourceMap)
|
||||
updateDB(margeResults)
|
||||
println "修改完成 "+new Date().toString()
|
||||
}
|
||||
|
||||
def megerDatas(sourceMap){
|
||||
def set = new HashSet()
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
set.addAll(valMap.keySet())
|
||||
}
|
||||
def lastValues = []
|
||||
set.each {id->
|
||||
|
||||
def dbObj = CommonDatautils.findDistrictById(id)
|
||||
|
||||
def datas = []
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
def sourceEsObj = findCommunities(valMap.get(id))
|
||||
if(sourceEsObj){
|
||||
if(source == "链家"){
|
||||
sourceEsObj.source = "lianjia"
|
||||
}else if(source == "我爱我家"){
|
||||
sourceEsObj.source = "wiwj"
|
||||
}else if(source == "安居客"){
|
||||
sourceEsObj.source = "anjuke"
|
||||
}
|
||||
datas << sourceEsObj
|
||||
}
|
||||
}
|
||||
if(dbObj && datas){
|
||||
def newValMap = getNewField2(datas, dbObj)
|
||||
if(newValMap){
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
statMap.each {source,fieldMap->
|
||||
println source +" "+fieldMap.size()
|
||||
fieldMap.each{f,v->
|
||||
println f+" "+ v
|
||||
}
|
||||
println "----------------------------------------"
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
|
||||
|
||||
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||
// def lianjisIds = lianjiaMap.keySet()
|
||||
// def wiwjIds = wiwjMap.keySet()
|
||||
// def anjukeIds = anjukeMap.keySet()
|
||||
// def set = new HashSet()
|
||||
// set.addAll(lianjisIds)
|
||||
// set.addAll(wiwjIds)
|
||||
// set.addAll(anjukeIds)
|
||||
//
|
||||
// def lastValues = []
|
||||
// def lianjiaUpdateCount = 0
|
||||
// def wiwjUpdateCount = 0
|
||||
// def anjukeUpdateCount = 0
|
||||
// def lianjiaCountMap = [:]
|
||||
// def wiwjCountMap = [:]
|
||||
// def anjukeCountMap = [:]
|
||||
// set.each {id->
|
||||
// BasicDBObject newValMap = new BasicDBObject()
|
||||
// fields.each {field->
|
||||
// def lianjiaValMap = lianjiaMap.get(id)
|
||||
// def wiwjValMap = wiwjMap.get(id)
|
||||
// def anjukeValMap = anjukeMap.get(id)
|
||||
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||
// newValMap.put(field, lianjiaValMap.get(field))
|
||||
// lianjiaUpdateCount++
|
||||
// def count = lianjiaCountMap.get(field)
|
||||
// if(!count){
|
||||
// lianjiaCountMap.put(field, 1)
|
||||
// }else{
|
||||
// lianjiaCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||
// newValMap.put(field, wiwjValMap.get(field))
|
||||
// wiwjUpdateCount++
|
||||
// def count = wiwjCountMap.get(field)
|
||||
// if(!count){
|
||||
// wiwjCountMap.put(field, 1)
|
||||
// }else{
|
||||
// wiwjCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||
// newValMap.put(field, anjukeValMap.get(field))
|
||||
// anjukeUpdateCount++
|
||||
// def count = anjukeCountMap.get(field)
|
||||
// if(!count){
|
||||
// anjukeCountMap.put(field, 1)
|
||||
// }else{
|
||||
// anjukeCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// newValMap.put("id", id)
|
||||
// lastValues << newValMap
|
||||
// }
|
||||
//
|
||||
// lianjiaCountMap.each {k,v->
|
||||
// println "[链家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// wiwjCountMap.each {k,v->
|
||||
// println "[我爱我家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// anjukeCountMap.each {k,v->
|
||||
// println "[安居客] "+k+" "+v
|
||||
// }
|
||||
// return lastValues
|
||||
// }
|
||||
|
||||
def updateDB(margeResults){
|
||||
for(def mr : margeResults){
|
||||
def id = mr.get("id")
|
||||
mr.remove("id")
|
||||
mr.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
|
||||
}
|
||||
}
|
||||
|
||||
// def batchUpdateDist(source){
|
||||
//// def rets = findThirdMatchDist(source)
|
||||
// println "[${source}] 符保条件小区量:"+rets.size()
|
||||
// def updateIndex = new AtomicLong()
|
||||
// final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
// def allUpdateList = new CopyOnWriteArrayList()
|
||||
//
|
||||
// def sourceMap = new ConcurrentHashMap()
|
||||
//
|
||||
// for(def i=0;i<rets.size();i++) {
|
||||
// final def ret = rets.get(i)
|
||||
//// fixedThreadPool.execute(
|
||||
//// new Runnable() {
|
||||
//// void run() {
|
||||
// try {
|
||||
// def newValMap = running(ret.url, ret.old_match_id)
|
||||
// if(newValMap){
|
||||
// sourceMap.put(ret.old_match_id, newValMap)
|
||||
// allUpdateList << newValMap
|
||||
// updateIndex.addAndGet(1)
|
||||
// }
|
||||
// }catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
//// }
|
||||
//// }
|
||||
//// )
|
||||
// }
|
||||
// latch.await()
|
||||
// println "总修改量:"+updateIndex.get()
|
||||
// def countMap = [:]
|
||||
// for(def map : allUpdateList){
|
||||
// map.each{k,v->
|
||||
// def count = countMap.get(k)
|
||||
// if(!count){
|
||||
// countMap.put(k, 1)
|
||||
// }else{
|
||||
// countMap.put(k, count+1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// countMap.each {k,v->
|
||||
// println k +" 更新量:"+v
|
||||
// }
|
||||
//
|
||||
// println "---------------------------------------------------------"
|
||||
// return sourceMap
|
||||
// }
|
||||
|
||||
def running(url, old_match_id){
|
||||
def esObj = findCommunities(url)
|
||||
def dbObj = CommonDatautils.findDistrictById(old_match_id)
|
||||
if(dbObj){
|
||||
def newValMap = getNewField(esObj, dbObj)
|
||||
return newValMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getNewField(esObj, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
esObj.each{field, esVal->
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(esVal && field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
|
||||
}
|
||||
if(esVal && field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(esVal && field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
newValMap.put(field, newVal)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return newValMap
|
||||
}
|
||||
|
||||
def getNewField2(esDatas, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
def lastValueMap = [:]
|
||||
def haveValueMap = [:]
|
||||
fields.each{field->
|
||||
for(def esData : esDatas){
|
||||
def esVal = esData.get(field)
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(haveValueMap.containsKey(field)){
|
||||
continue
|
||||
}else{
|
||||
if(lastValueMap.containsKey(field)){
|
||||
continue
|
||||
}
|
||||
if(esVal){
|
||||
|
||||
if(field=="estate_charge" || field=="car_site_count") {
|
||||
esVal = FiledStrand.getNumber(esVal)
|
||||
}
|
||||
if(dbVal && esVal.toString()==dbVal.toString()){
|
||||
haveValueMap.put(field, esVal)
|
||||
}else{
|
||||
if(field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
lastValueMap.put(field, newVal)
|
||||
|
||||
//以下为统计各网站各字段的更新情况
|
||||
def fieldStatMap = statMap.get(esData.source)
|
||||
if(!fieldStatMap){
|
||||
fieldStatMap = [:]
|
||||
statMap.put(esData.source, fieldStatMap)
|
||||
}
|
||||
def count = fieldStatMap.get(field)
|
||||
if(!count){
|
||||
fieldStatMap.put(field, 1)
|
||||
}else{
|
||||
fieldStatMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValueMap
|
||||
}
|
||||
|
||||
def findThirdMatchDist(source){
|
||||
def query = new BasicDBObject("source",source).append("old_match_way", 1)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("url", 1)
|
||||
keys.put("old_match_id", 1)
|
||||
keys.put("old_match_city", 1)
|
||||
keys.put("old_match_district", 1)
|
||||
keys.put("old_match_name", 1)
|
||||
def idsMap = [:]
|
||||
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
|
||||
for(def db : dbcursor){
|
||||
if(!db.old_match_id && db.old_match_name){
|
||||
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
|
||||
if(dbo){
|
||||
idsMap.put(dbo._id.toString(), db.url)
|
||||
}
|
||||
}else{
|
||||
idsMap.put(db.old_match_id, db.url)
|
||||
}
|
||||
|
||||
}
|
||||
return idsMap
|
||||
}
|
||||
|
||||
static def findCommunities(url){
|
||||
if(!url){
|
||||
return null
|
||||
}
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||
def rets = []
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
}catch(Exception e){
|
||||
findCommunities(url)
|
||||
}
|
||||
def data = [:]
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
data.put("household_count", d.households)
|
||||
data.put("loudong_count", d.loudong_cnt)
|
||||
data.put("developer", d.developer)
|
||||
data.put("estate_company", d.property_company)
|
||||
data.put("estate_charge", d.property_fee)
|
||||
data.put("car_site_count", d.parking_cnt)
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
+75
-1
@@ -26,7 +26,8 @@ class EsMonthHouseStat {
|
||||
// task.statHouseUpdateCount()
|
||||
|
||||
// task.statDistHouse()
|
||||
task.statRent()
|
||||
// task.statRent()
|
||||
task.statOfficeHouse()
|
||||
}
|
||||
|
||||
def statRent(){
|
||||
@@ -201,6 +202,51 @@ class EsMonthHouseStat {
|
||||
},0)
|
||||
}
|
||||
|
||||
def statOfficeHouse(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allLists = []
|
||||
allLists << ["城市", "近一年写字楼出售量", "写字楼量", "近一年出售房源量大于3的写字楼量", "大于3写字楼量占比"]
|
||||
citys.each{c->
|
||||
def count = findHouseCountByCat(c, "office_sell",20240101, 20250201)
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = 0
|
||||
if(count){
|
||||
distCount = findHouseCountByCat(dist.city, dist.region, alias, 20240101, 20250201, "office_sell")
|
||||
if(distCount && distCount>=3){
|
||||
disthouseCount++
|
||||
}
|
||||
}
|
||||
// println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
// println districts.size()
|
||||
def precent = 0
|
||||
if(districts){
|
||||
precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
}
|
||||
def row = [c , count, districts.size(), disthouseCount, precent+"%"]
|
||||
println row
|
||||
allLists << row
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\全国写字楼情况占比统计.xlsx", allLists, "写字楼情况")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
@@ -219,6 +265,23 @@ class EsMonthHouseStat {
|
||||
return total
|
||||
}
|
||||
|
||||
def findHouseCountByCat(def city, def region, def alias, def startDate, def endDate, cat){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
@@ -240,6 +303,17 @@ class EsMonthHouseStat {
|
||||
}
|
||||
|
||||
|
||||
def findHouseCountByCat(def city, def cat, def startDate, def endDate){
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+73
-2
@@ -32,8 +32,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
|
||||
"淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12","房多多":"13","我爱我家":"14",
|
||||
"禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20",
|
||||
"乐有家":"21","幸福里":"22","中国房价行情":"23","常州住房":"24"
|
||||
]
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
@@ -269,6 +272,33 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
if(third.get("csjj_esf_price")){
|
||||
price_map.csjj_avg_price = third.get("csjj_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("leyoujia_esf_price")){
|
||||
price_map.leyoujia_avg_price = third.get("leyoujia_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("c21_esf_price")){
|
||||
price_map.c21_avg_price = third.get("c21_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("jiwu_esf_price")){
|
||||
price_map.jiwu_avg_price = third.get("jiwu_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("xingfuli_esf_price")){
|
||||
price_map.xingfuli_avg_price = third.get("xingfuli_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("zwfc_esf_price")){
|
||||
price_map.zwfc_avg_price = third.get("zwfc_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("creprice_esf_price")){
|
||||
price_map.creprice_avg_price = third.get("creprice_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -317,6 +347,20 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "ÎÒ°®ÎÒ¼Ò"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "·¿²ú³¬ÊÐ"
|
||||
}else if(k=="csjj_avg_price"){
|
||||
web = "城市居家"
|
||||
}else if(k=="leyoujia_avg_price"){
|
||||
web = "乐有家"
|
||||
}else if(k=="c21_avg_price"){
|
||||
web = "C21"
|
||||
}else if(k=="jiwu_avg_price"){
|
||||
web = "吉屋网"
|
||||
}else if(k=="xingfuli_avg_price"){
|
||||
web = "幸福里"
|
||||
}else if(k=="creprice_avg_price"){
|
||||
web = "中国房价行情"
|
||||
}else if(k=="zwfc_avg_price"){
|
||||
web = "常州住房"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -470,6 +514,33 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
|
||||
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("leyoujia_esf_price") && db.get("leyoujia_esf_price") && db.get("leyoujia_esf_price")>3000){
|
||||
dataMap.put("leyoujia_esf_price", db.get("leyoujia_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("c21_esf_price") && db.get("c21_esf_price") && db.get("c21_esf_price")>3000){
|
||||
dataMap.put("c21_esf_price", db.get("c21_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("jiwu_esf_price") && db.get("jiwu_esf_price") && db.get("jiwu_esf_price")>3000){
|
||||
dataMap.put("jiwu_esf_price", db.get("jiwu_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("xingfuli_esf_price") && db.get("xingfuli_esf_price") && db.get("xingfuli_esf_price")>3000){
|
||||
dataMap.put("xingfuli_esf_price", db.get("xingfuli_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("creprice_esf_price") && db.get("creprice_esf_price") && db.get("creprice_esf_price")>3000){
|
||||
dataMap.put("creprice_esf_price", db.get("creprice_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("zwfc_esf_price") && db.get("zwfc_esf_price") && db.get("zwfc_esf_price")>3000){
|
||||
dataMap.put("zwfc_esf_price", db.get("zwfc_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+52
-25
@@ -38,8 +38,8 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
def citys = CommonUtils.findAllCity()
|
||||
citys.each {city->
|
||||
def communitys = findAllEsData2(city, mdate)
|
||||
@@ -47,14 +47,13 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
//第三方匹配小区
|
||||
matchCommunitys(communitys)
|
||||
}
|
||||
|
||||
statImgCount()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findAllEsData2(city, mdate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
|
||||
//安居客小程序
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "贝壳找房"))
|
||||
.must(date_range).must(date_range)
|
||||
|
||||
def rets = []
|
||||
@@ -106,41 +105,69 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
def total = communitys.size()
|
||||
for(def i=0;i<communitys.size();i++){
|
||||
final def d = communitys[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
||||
|
||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","贝壳找房").append("match_id", new BasicDBObject('$exists', true))
|
||||
def obj = thirdSeaweedMappingColl.findOne(query)
|
||||
if(obj) {
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images){
|
||||
vo.put("out_pics", d.out_images)
|
||||
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
|
||||
if(obj.old_match_id && obj.match_id==obj.old_match_id){
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images && (!imageObj || (imageObj && !imageObj.out_pics))){
|
||||
vo.put("out_pics", d.out_images)
|
||||
}
|
||||
if(d.house_type_images && (!imageObj || (imageObj && !imageObj.floorplan_pics))){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
if(vo.out_pics || vo.floorplan_pics){
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}
|
||||
}
|
||||
if(d.house_type_images){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}else{
|
||||
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def statImgCount(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = seaweed_fs_coll.find(q)
|
||||
def out_pics_count = 0
|
||||
def floor_count = 0
|
||||
for(def d : dbcursor){
|
||||
if(d.out_pics){
|
||||
out_pics_count = out_pics_count + d.out_pics.size()
|
||||
}
|
||||
if(d.floorplan_pics){
|
||||
d.floorplan_pics.each{k,v->
|
||||
floor_count = floor_count + v.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
println out_pics_count
|
||||
println floor_count
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+15
-15
@@ -13,18 +13,18 @@ class MsbPocTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\房讯通10月价格核查_1126-FXT提交结果20241205.xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
// matchOld(inputfile)
|
||||
matchOld(inputfile)
|
||||
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
|
||||
// test("D:\\数据\\美术宝\\test2.xlsx")
|
||||
// test("D:\\数据\\美术宝\\FiveLevelTest.xlsx")
|
||||
|
||||
// cityPrice()
|
||||
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
|
||||
|
||||
addCCBPrice(inputfile)
|
||||
// addCCBPrice(inputfile)
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
@@ -69,24 +69,24 @@ class MsbPocTest {
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Y'] && row['G']){
|
||||
if(!row['AD'] ){
|
||||
def city = row['C']
|
||||
def region = null //row['D']
|
||||
def name = row['E']
|
||||
def address = row['G']
|
||||
def name = null// row['E']
|
||||
def address = row['D']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
row['AC'] = jsonObject.get("propertyType")
|
||||
row['AD'] = jsonObject.get("id")
|
||||
row['AE'] = jsonObject.get("city")
|
||||
row['AF'] = jsonObject.get("district")
|
||||
row['AG'] = jsonObject.get("name")
|
||||
row['AH'] = jsonObject.get("propertyType")
|
||||
// row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
@@ -114,7 +114,7 @@ class MsbPocTest {
|
||||
if(!row['Y']){
|
||||
def city = row['C']
|
||||
def region = null //row['B']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
def address = row['D'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
@@ -125,7 +125,7 @@ class MsbPocTest {
|
||||
row['AB'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+23
-21
@@ -38,9 +38,9 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
def filePath = "D:\\Êý¾Ý\\¼Û¸ñ\\»ÝÖÝ.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
@@ -174,26 +174,28 @@ class AddPriceTest {
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['U']){
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(d['A']){
|
||||
def cityId = cityCodeDataMap.get(d['B'])
|
||||
def map = findAvgPrice(d['A'], cityId)
|
||||
if(map){
|
||||
d['H'] = map.avg_price
|
||||
d['I'] = map.calc_type
|
||||
d['I'] = map.avg_price
|
||||
d['J'] = map.month
|
||||
// d['H'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
|
||||
// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId)
|
||||
// if(newPriceMap){
|
||||
// d['Q'] = newPriceMap.avg_price
|
||||
// }
|
||||
def newPriceMap = findNewFjwAvgPrice(d['A'], cityId)
|
||||
if(newPriceMap){
|
||||
d['G'] = newPriceMap.avg_price
|
||||
d['H'] = newPriceMap.month
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -213,7 +215,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findNewFjwAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance235.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -270,14 +272,14 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['B']) {
|
||||
if (!d['H']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
if(gxdPrice){
|
||||
d['B'] = gxdPrice.price
|
||||
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
|
||||
d['H'] = gxdPrice.price
|
||||
d['I'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -305,11 +307,11 @@ class AddPriceTest {
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
// if(!d['L']){
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['AJ'])
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
d['E'] = dist.name
|
||||
// d['E'] = dist.name
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.name_cat
|
||||
// d['F'] = dist.estate_type2
|
||||
@@ -318,9 +320,9 @@ class AddPriceTest {
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['G'] = dbo.avg_price
|
||||
d['E'] = dbo.avg_price_2b
|
||||
// d['X'] = dbo.cal_type
|
||||
// d['E'] = dbo.avg_price_2b
|
||||
// d['E'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
// }
|
||||
|
||||
+5
-2
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -17,7 +18,7 @@ import com.mongodb.DBCollection
|
||||
*/
|
||||
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
//@BatchOwner(Watcher.GaoJuan)
|
||||
class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
|
||||
@@ -69,7 +70,9 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
if(r.lon && r.lat){
|
||||
for(def m : zimaoqu_range_datas){
|
||||
// def flag = InsidePolygon.isInArea(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
|
||||
def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
|
||||
List<List<Point>> pointsList = InsidePolygon.createPoint4(m.shape)
|
||||
// def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, pointsList)
|
||||
def flag = InsidePolygon.isInArea(r.lon, r.lat, pointsList)
|
||||
if(flag){
|
||||
resKey = m.name
|
||||
break
|
||||
|
||||
+18
-16
@@ -226,16 +226,17 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
for(def key : dist.mapping_alias){
|
||||
if(key && !key.toString().equals("null")){
|
||||
aliasNames << key
|
||||
if(aliasNames.size()>5){
|
||||
map.alias = aliasNames
|
||||
break;
|
||||
}else{
|
||||
if(aliasNames){
|
||||
map.alias = aliasNames
|
||||
}
|
||||
}
|
||||
// if(aliasNames.size()>5){
|
||||
// map.alias = aliasNames
|
||||
// break;
|
||||
// }else{
|
||||
// if(aliasNames){
|
||||
// map.alias = aliasNames
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
map.alias = aliasNames
|
||||
}
|
||||
|
||||
if(dist.addr_alias){
|
||||
@@ -243,16 +244,17 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
for(def key : dist.addr_alias){
|
||||
if(key && !key.toString().equals("null")){
|
||||
names << key
|
||||
if(names.size()>5){
|
||||
map.addr_alias = names
|
||||
break;
|
||||
}else{
|
||||
if(names){
|
||||
map.addr_alias = names
|
||||
}
|
||||
}
|
||||
// if(names.size()>5){
|
||||
// map.addr_alias = names
|
||||
// break;
|
||||
// }else{
|
||||
// if(names){
|
||||
// map.addr_alias = names
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
map.addr_alias = names
|
||||
}
|
||||
|
||||
map.category = dist.cat
|
||||
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
package com.fangjia.script.looplineinfo
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
|
||||
class InPolygon {
|
||||
|
||||
|
||||
public static List<List<Point2D.Double>> createPoint(def pointList){
|
||||
List<List<Point2D.Double>> alllist = new ArrayList<>();
|
||||
for(def pointArr : pointList){
|
||||
List<Point2D.Double> list = new ArrayList<>();
|
||||
for(def arr : pointArr){
|
||||
double x = DataHandleUtils.getDoubleRound(6, arr[0]);
|
||||
double y = DataHandleUtils.getDoubleRound(6, arr[1]);
|
||||
Point2D.Double point = new Point2D.Double(x,y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
public static def isInArea(Point2D.Double point, List<List<Point2D.Double>> pointList){
|
||||
def flag = false;
|
||||
for(List<Point2D.Double> pointArr : pointList){
|
||||
flag = isPtInPoly(point, pointArr)
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断点是否在多边形内
|
||||
* <p>
|
||||
* 整个算法的思路为:作点平行于y轴的射线,这样就可以直接比较交点是否大于点(x,y)中y的值,可以简化判断。
|
||||
*
|
||||
* @param point 检测点
|
||||
* @param pts 多边形的顶点
|
||||
* @return 点在多边形内返回true, 否则返回false
|
||||
*/
|
||||
public static boolean isPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
|
||||
int polygonSidesCount = pts.size();
|
||||
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
|
||||
boolean boundOrVertex = true;
|
||||
// 射线与多边形相交的次数
|
||||
int intersectCount = 0;
|
||||
//浮点类型计算时候与0比较时候的容差
|
||||
double precision = 2e-10;
|
||||
// 相邻两点
|
||||
Point2D.Double p1, p2;
|
||||
// 当前点
|
||||
Point2D.Double p = point;
|
||||
// 线段起点
|
||||
p1 = pts.get(0);
|
||||
for (int i = 1; i <= polygonSidesCount; ++i) {
|
||||
//按照点录入的顺序依次检查相邻两点组成的线段和当前的点的关系。
|
||||
if (p.equals(p1)) {
|
||||
//如果当前点就是多边形的顶点之一,直接返回。
|
||||
return boundOrVertex;
|
||||
}
|
||||
// 线段终点
|
||||
p2 = pts.get(i % polygonSidesCount);
|
||||
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
|
||||
//点在x轴上的映射,明显超出了线段在x轴上的投影。此时冲着上做射线,肯定没有交点,直接跳过。
|
||||
p1 = p2;
|
||||
continue;
|
||||
}
|
||||
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
|
||||
// 当前点在线段于x轴上的投影内
|
||||
if (p.y <= Math.max(p1.y, p2.y)) {
|
||||
//当前点的 y 坐标小于 线段对y轴投影的最大值
|
||||
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
|
||||
// 若线段同样是平行于y轴,则可以断定,当前点,在多边形的这条垂直于x轴的边上。
|
||||
return boundOrVertex;
|
||||
}
|
||||
if (p1.y == p2.y) {
|
||||
// 若线段为平行于x轴的水平线
|
||||
if (p1.y == p.y) {
|
||||
//当前点正好位于该水平线上,直接返回该点位于多边形的一条边上。
|
||||
return boundOrVertex;
|
||||
} else {
|
||||
//如果当前点在水平线以下,增加一个交点。
|
||||
++intersectCount;
|
||||
}
|
||||
} else {
|
||||
// 如果不是水平线,则用两点式求当前点的x带入多边形线段的直线方程后,对应的y的坐标
|
||||
double xInSideLineFormulaResultY = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
|
||||
if (Math.abs(p.y - xInSideLineFormulaResultY) < precision) {
|
||||
// 误差允许范围内,改点就在线段上,则表明,该点位于多边形的一个边上。
|
||||
return boundOrVertex;
|
||||
}
|
||||
|
||||
if (p.y < xInSideLineFormulaResultY) {
|
||||
// 如果线段上取得的y比当前点的y要大,当前做向上的射线,肯定交于上方的一个点。
|
||||
++intersectCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 当前点不在线段投影到x轴的区间中
|
||||
if (p.x == p2.x && p.y <= p2.y) {
|
||||
// 但恰好位于线段终点的x坐标对应的平行于y轴上的线上的低于终点y的一点
|
||||
// 此时检查下一点能否将其x边界包含。
|
||||
Point2D.Double p3 = pts.get((i + 1) % polygonSidesCount);
|
||||
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
|
||||
// 若当前点的x坐标位于 p1和p3组成的线段关于x轴的投影中,则记为该点的射线只穿过端点一次。
|
||||
++intersectCount;
|
||||
} else {
|
||||
// 若当前点的x坐标不能包含在p1和p3组成的线段关于x轴的投影中,则点射线通过的两条线段组成了一个弯折的部分,
|
||||
// 此时我们记射线穿过该端点两次
|
||||
intersectCount += 2;
|
||||
}
|
||||
// 此判断的核心思路是由点在两条线段的内部还是外部去思考得出的
|
||||
}
|
||||
}
|
||||
// 进行下一个线段的判断
|
||||
p1 = p2;
|
||||
}
|
||||
if (intersectCount % 2 == 0) {
|
||||
//偶数在多边形外
|
||||
return false;
|
||||
} else {
|
||||
//奇数在多边形内
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+4
@@ -324,4 +324,8 @@ class InsidePolygon {
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package com.fangjia.script.majintao
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
|
||||
String excelFilePath = "F:\\估家\\文件\\北交所\\testcopy1.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
String comprehensiveUrl = "http://localhost:9180/comprehensiveQuery/houseValueInformation";
|
||||
String communityUrl = "http://open.fangjia.com/addrSvc/avgPrice";
|
||||
String valueUrl = "http://open.fangjia.com/property/addressEvaluate";
|
||||
String caseUrl = "http://open.fangjia.com/house/case";
|
||||
String poiUrl = "http://open.fangjia.com/district/addrFacilitys";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
String city = items['A']
|
||||
String address = items['B']
|
||||
double size = items['C']
|
||||
|
||||
System.out.println(city + "---" + address + "----" + size);
|
||||
|
||||
map.put("city", city)
|
||||
map.put("address", address)
|
||||
map.put("size", size.toString())
|
||||
map.put("strctureArea", size.toString())
|
||||
map.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB")
|
||||
|
||||
if (!address.contains("%")) {
|
||||
// 发送HTTP请求
|
||||
long start
|
||||
long end
|
||||
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(communityUrl, map, "UTF-8")
|
||||
//
|
||||
//
|
||||
// end = System.currentTimeMillis();
|
||||
// items['D'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(valueUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['E'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(caseUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['F'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(poiUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['G'] = String.valueOf(end - start);
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
def content = httpRequest.getJsonContent(comprehensiveUrl, map, "UTF-8")
|
||||
end = System.currentTimeMillis();
|
||||
items['H'] = String.valueOf(end - start);
|
||||
println content.get("msg")
|
||||
if(content.get("msg").contains("该地址不支持估价!")) {
|
||||
items['I'] = "该地址不支持估价!"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
a = new ObjectId()
|
||||
print(a)
|
||||
+6
-18
@@ -1,28 +1,16 @@
|
||||
package com.fangjia.script.trade
|
||||
|
||||
|
||||
import java.util.regex.Matcher
|
||||
|
||||
package com.fangjia.script.sync
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.script.BaseBatch;
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBObject
|
||||
import com.fangjia.script.util.MongoUtils;
|
||||
import com.fangjia.core.util.MongoUtils;
|
||||
import com.fangjia.script.Watcher;
|
||||
import com.mongodb.Bytes
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
/**
|
||||
* ½«ÏßÉÏopen uriÊý¾Ýͬ²½µ½192.168.0.60ÉÏ
|
||||
* @author gaojuan
|
||||
@@ -31,7 +19,7 @@ import org.apache.commons.io.FileUtils
|
||||
class CopyOpenUriLog extends BaseBatch{
|
||||
|
||||
|
||||
static DBCollection DESC_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com", 27020)
|
||||
static DBCollection DESC_OPEN = MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
public static final DBCollection SOURCE_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "log.mongo.db.fangjia.com", 52002)
|
||||
static main(String[] args){
|
||||
runBatch(new CopyOpenUriLog(), args)
|
||||
+1
@@ -1062,6 +1062,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
q.put("s_date", SHORT_DATE);
|
||||
|
||||
DBObject udata2 = new BasicDBObject()
|
||||
udata2.put("match_id", district._id.toString());
|
||||
udata2.put("match_city", district.city);
|
||||
udata2.put("match_region", district.region);
|
||||
udata2.put("match_name", district.name);
|
||||
|
||||
+3
-1
@@ -27,7 +27,9 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||
// def esSources = ["搜房":"搜房"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
|
||||
def esSources = ["安居客小程序":"安居客","搜房":"搜房","诸葛找房APP":"诸葛找房","幸福里APP":"幸福里","吉屋":"吉屋","中国房价行情":"中国房价行情","贝壳找房":"贝壳找房",
|
||||
"链家":"链家","C21":"C21","城市房产小程序":"城市房产","乐有家":"乐有家","我爱我家":"我爱我家","中原地产":"中原地产","兔博士PC端":"兔博士",
|
||||
"365淘房":"365淘房","Q房":"Q房","麦田":"麦田","常州住房":"常州住房"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDistrictPrice(), args);
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
|
||||
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
|
||||
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf", "房天下":"fang_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
+33
-9
@@ -9,13 +9,8 @@ import net.sf.json.JSONObject
|
||||
|
||||
class BaiduMapUtils {
|
||||
|
||||
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
|
||||
def static AKS = ["hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
|
||||
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
|
||||
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
|
||||
"B42b12d01f2dee712fd66c3e886158a4", "1a3cde429f38434f1811a75e1a90310c","W3M5uHfcrmlAdhQjf6v93LCI",
|
||||
"C8bb035aacf602551e829e4c64575224", "B42b12d01f2dee712fd66c3e886158a4",
|
||||
"1a3cde429f38434f1811a75e1a90310c","9E99D3540e4dafb7894f7b9fe27fa6b5"];
|
||||
private static String AK = "1a3cde429f38434f1811a75e1a90310c";
|
||||
def static AKS = ["1a3cde429f38434f1811a75e1a90310c","1a3cde429f38434f1811a75e1a90310c"];
|
||||
public static boolean HAS_AVAILABLE_AK = true;
|
||||
|
||||
static def getAddress(lng, lat) throws Exception{
|
||||
@@ -83,8 +78,8 @@ class BaiduMapUtils {
|
||||
throw new RuntimeException("无可用百度AK");
|
||||
}
|
||||
System.out.println("更换AK");
|
||||
for(int i = 0 ;i < AKS.length; i++){
|
||||
if(i == AKS.length - 1){
|
||||
for(int i = 0 ;i < AKS.size(); i++){
|
||||
if(i == AKS.size() - 1){
|
||||
AK = AKS[0];
|
||||
HAS_AVAILABLE_AK = false;
|
||||
throw new RuntimeException("无可用百度AK");
|
||||
@@ -127,6 +122,35 @@ class BaiduMapUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject geocoding(locationstr) {
|
||||
String url = "https://api.map.baidu.com/reverse_geocoding/v3/?ak="+AK+"&output=json&coordtype=gcj02ll&location="+locationstr
|
||||
boolean stop = false;
|
||||
int recount = 0;
|
||||
JSONObject jsonObj = null
|
||||
while(!stop && recount<6) {
|
||||
try{
|
||||
HttpRequest request = new HttpRequest();
|
||||
try{
|
||||
String result = request.get(url,null,"UTF-8");
|
||||
def json = JSONObject.fromObject(result);
|
||||
if(json.getInt("status") >= 300){
|
||||
changeAK();
|
||||
recount++
|
||||
}else{
|
||||
jsonObj = json.result
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}catch (Exception e){
|
||||
recount++;
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
public static def searchByBounds(key, min_lat, min_lng, max_lat, max_lng){
|
||||
boolean stop = false;
|
||||
int recount = 0;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
@@ -192,6 +193,19 @@ public class CommonUtils {
|
||||
}
|
||||
}
|
||||
|
||||
static void recalcAverage(String city, String region, String name, Integer count =1 , String category="sell", Boolean retainHistory=true){
|
||||
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate";
|
||||
def params = new HashMap<String,String>();
|
||||
params.city = city;
|
||||
params.region = region;
|
||||
params.name = name;
|
||||
params.count = count.toString();
|
||||
params.category = category;
|
||||
params.retainHistory = retainHistory.toString();
|
||||
HttpRequest request = new HttpRequest();
|
||||
println(request.get(url, params, "UTF-8"))
|
||||
}
|
||||
|
||||
/**
|
||||
* 重算小区均价
|
||||
* @param city
|
||||
@@ -221,7 +235,7 @@ public class CommonUtils {
|
||||
cmd.put("name", name)
|
||||
cmd.put('estate_type', estateType)
|
||||
cmd.put("calc_date", calcDate)
|
||||
cmd.put("calc_type", 0)
|
||||
cmd.put("calc_type", calcType)
|
||||
println(cmd)
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("category", "sell")
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
|
||||
import java.text.DecimalFormat
|
||||
|
||||
class GeoHelper {
|
||||
|
||||
|
||||
@@ -60,10 +64,159 @@ class GeoHelper {
|
||||
// return GetLatLon(a, distance, angle);
|
||||
// }
|
||||
//
|
||||
public static void main(String[] args){
|
||||
GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
|
||||
GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
|
||||
}
|
||||
public static haversine(lat1, lon1, lat2, lon2) {
|
||||
def R = 6371; // 地球半径(公里)
|
||||
def lat1Rad = lat1 * (Math.PI / 180);
|
||||
def lat2Rad = lat2 * (Math.PI / 180);
|
||||
def deltaLat = (lat2 - lat1) * (Math.PI / 180);
|
||||
def deltaLon = (lon2 - lon1) * (Math.PI / 180);
|
||||
|
||||
def a = Math.sin(deltaLat / 2) ** 2 +
|
||||
Math.cos(lat1Rad) * Math.cos(lat2Rad) *
|
||||
Math.sin(deltaLon / 2) ** 2;
|
||||
|
||||
def c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
||||
return R * c; // 返回距离(公里)
|
||||
}
|
||||
|
||||
public static calculateArea(coords) {
|
||||
def area = 0;
|
||||
def n = coords.size();
|
||||
|
||||
for (def i = 0; i < n; i++) {
|
||||
def j = (i + 1) % n;
|
||||
area += coords[i][0] * coords[j][1];
|
||||
area -= coords[j][0] * coords[i][1];
|
||||
}
|
||||
area = Math.abs(area) / 2; // 平方度
|
||||
|
||||
// 转换为平方公里
|
||||
def conversionFactor = 111.32; // 大约每度经纬度的公里数(考虑到地球的曲率)
|
||||
|
||||
def df = new DecimalFormat("#.000000")
|
||||
return Double.parseDouble(df.format(area * conversionFactor * conversionFactor)) // 返回面积(平方公里)
|
||||
// return area * conversionFactor * conversionFactor
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args){
|
||||
// GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
|
||||
// GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
|
||||
|
||||
List<List<Double>> coordinates = [
|
||||
[
|
||||
117.61878540582734,
|
||||
24.667331307258223
|
||||
],
|
||||
[
|
||||
117.61400840589488,
|
||||
24.670008244935104
|
||||
],
|
||||
[
|
||||
117.60936140565889,
|
||||
24.671837202755682
|
||||
],
|
||||
[
|
||||
117.60834440585768,
|
||||
24.668667276729821
|
||||
],
|
||||
[
|
||||
117.60696440583342,
|
||||
24.667256309201207
|
||||
],
|
||||
[
|
||||
117.60517040553236,
|
||||
24.667762297657362
|
||||
],
|
||||
[
|
||||
117.60272340547806,
|
||||
24.660100475272998
|
||||
],
|
||||
[
|
||||
117.60254340555321,
|
||||
24.658630509428463
|
||||
],
|
||||
[
|
||||
117.60206940590035,
|
||||
24.654781598282568
|
||||
],
|
||||
[
|
||||
117.60254840557604,
|
||||
24.652731645907636
|
||||
],
|
||||
[
|
||||
117.60346440599301,
|
||||
24.651555673447035
|
||||
],
|
||||
[
|
||||
117.6070314057805,
|
||||
24.652578649754957
|
||||
],
|
||||
[
|
||||
117.61605140571805,
|
||||
24.651939664372833
|
||||
],
|
||||
[
|
||||
117.6156454058372,
|
||||
24.654165613124405
|
||||
],
|
||||
[
|
||||
117.61559340595863,
|
||||
24.655023593107103
|
||||
],
|
||||
[
|
||||
117.61535440558418,
|
||||
24.658990501065318
|
||||
],
|
||||
[
|
||||
117.61662340528017,
|
||||
24.659352492642654
|
||||
],
|
||||
[
|
||||
117.61678740531192,
|
||||
24.661365445721003
|
||||
],
|
||||
[
|
||||
117.61472940542001,
|
||||
24.661623439403797
|
||||
],
|
||||
[
|
||||
117.61462940586114,
|
||||
24.661837434759093
|
||||
],
|
||||
[
|
||||
117.61470140583106,
|
||||
24.662665415879701
|
||||
],
|
||||
[
|
||||
117.61487240553515,
|
||||
24.663165403873506
|
||||
],
|
||||
[
|
||||
117.61522940591078,
|
||||
24.663494396452791
|
||||
],
|
||||
[
|
||||
117.6157014059138,
|
||||
24.663523395394918
|
||||
],
|
||||
[
|
||||
117.61627240565099,
|
||||
24.663438397829282
|
||||
],
|
||||
[
|
||||
117.61693540544979,
|
||||
24.663181403867725
|
||||
],
|
||||
[
|
||||
117.61878540582734,
|
||||
24.667331307258223
|
||||
]
|
||||
]
|
||||
|
||||
double area = calculateArea(coordinates)
|
||||
println "The area of the polygon is: ${String.format("%.2f", area)} square kilometers"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -43,15 +43,15 @@ class Test {
|
||||
|
||||
static main(args) {
|
||||
// String test1 = " qwdwqd ssss ";
|
||||
// String test2 = " qwdwqd ss";
|
||||
// String FiveLevelTest = " qwdwqd ss";
|
||||
// String test3 = "qwdwqd dd";
|
||||
//
|
||||
// test1 = test1.replaceAll(" |\\s+", " ");
|
||||
// test2 = test2.replaceAll(" |\\s+", " ");
|
||||
// FiveLevelTest = FiveLevelTest.replaceAll(" |\\s+", " ");
|
||||
// test3 = test3.replaceAll(" |\\s+", " ");
|
||||
//
|
||||
// String[] arrs1 = test1.split("\\s+")
|
||||
// String[] arrs2 = test2.split("\\s+")
|
||||
// String[] arrs2 = FiveLevelTest.split("\\s+")
|
||||
// String[] arrs3 = test3.split("\\s+")
|
||||
//
|
||||
// println arrs1
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package com.fangjia.script.wenyan
|
||||
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.action.get.GetRequest
|
||||
import org.elasticsearch.action.get.GetResponse
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
|
||||
////def data = []
|
||||
////data << ["ID","城市","区域","名称","地址", "财力评级"]
|
||||
//RestHighLevelClient client = ElasticUtils.getClient89("192.168.0.89")
|
||||
//district_analy_data_client = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data")
|
||||
//level_map = ["1": "S", "2": "A", "3": "B", "4": "C", "5": "D"]
|
||||
//level_maps = new HashMap<>()
|
||||
//for(def key in level_map.keySet()){
|
||||
// level_maps.put(level_map[key], key)
|
||||
//}
|
||||
//ExcelUtils.modify("C:\\Users\\goojia\\Desktop\\财力评级-厦门.xlsx",{List item->
|
||||
//// GetResponse response = client.get(new GetRequest("community_summary", item["A"] as String), RequestOptions.DEFAULT)
|
||||
//// def query = new Document("city", item['B'].toString().replace("市", ""))
|
||||
//// query.append("region", item['C'].replace("区", ""))
|
||||
//// query.append("district_name", item["D"])
|
||||
//// def stat = district_analy_data_client.find(query).first()
|
||||
//// println(query)
|
||||
//// item["F"] = response.getSourceAsMap()?.consumption_rating
|
||||
//// item["G"] = stat?.community_rating
|
||||
// item["G"] = level_maps.get(item["G"], null)
|
||||
//// println(item)
|
||||
//})
|
||||
//System.exit(0)
|
||||
////for (def region in ["思明", "翔安", "湖里"]) {
|
||||
//// DbUtils.seaweed.find(new BasicDBObject("city", "厦门").append("region", region).append("cat", "district").append("status", 0)).each { BasicDBObject dbo ->
|
||||
//// println("${dbo._id.toString()}\t${dbo.city}市\t${dbo.region}区\t${dbo.name}\t${dbo.address}")
|
||||
//// }
|
||||
////}
|
||||
|
||||
ExcelUtils.read("C:\\Users\\goojia\\Desktop\\海南省城市均价.xlsx",1).each {List item->
|
||||
def yearMonth = DateUtil.formatByPattern(item[0], "yyyyMM").toInteger()
|
||||
|
||||
println(item)
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.fangjia.script.wenyan.xiamen
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\本地化部署\\厦门银行数据\\网点经纬度业务选择网点.xlsx",{List item ->
|
||||
def location = item['E']
|
||||
if(location){
|
||||
def lng = location.toString().split(",")[0].toDouble()
|
||||
def lat = location.toString().split(",")[1].toDouble()
|
||||
def bounds = GeoUtils.getAround(lat, lng, 3000)
|
||||
item["G"] = "${DataUtil.getRound(6, bounds[2])},${DataUtil.getRound(6,bounds[0])},${DataUtil.getRound(6,bounds[3])},${DataUtil.getRound(6,bounds[1])}"
|
||||
}
|
||||
})
|
||||
+23
-6
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //±¦½£
|
||||
//94c74c60e102972a8d17b7b75d7403be
|
||||
def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //±¦½£:c6fcd4bfbc40d73ee762b8115dca3390
|
||||
//15021384995£ºdfc2245bac8a7b2d5b62e060e59d4eca
|
||||
|
||||
/**
|
||||
@@ -29,12 +30,28 @@ class GaoDeApiUtils {
|
||||
params.put("location", location)
|
||||
params.put("extensions","all")
|
||||
String content
|
||||
try {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
def stop = true
|
||||
JSONObject jsonObj = null
|
||||
def errorCount = 1
|
||||
while (stop){
|
||||
try {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
}
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
if(jsonObj.status!="1"){
|
||||
println jsonObj
|
||||
errorCount ++
|
||||
Thread.sleep(3000)
|
||||
if(errorCount>3){
|
||||
stop = false
|
||||
}
|
||||
}else{
|
||||
stop = false
|
||||
}
|
||||
}
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user