Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-03-17 15:13:06 +08:00
207 changed files with 17219 additions and 7547 deletions
+6
View File
@@ -537,6 +537,12 @@
<version>1.70</version> <!-- 确保使用最新版本 -->
</dependency>
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>1.18.2</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -78,7 +78,7 @@ class PoseidonOnce {
// DBCursor cur = coll.find()
// println cur.count()
String url = "http://www.fangjia.com";
DBCollection coll = MongoUtils.getCollectionByName("fangjia", "test2", "192.168.0.188")
DBCollection coll = MongoUtils.getCollectionByName("fangjia", "FiveLevelTest", "192.168.0.188")
// for(int i=426000;i<1000000;i++) {
// String s = url + i + "_" + url.hashCode()
// DBObject dbo = new BasicDBObject()
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -71,8 +71,7 @@ class BigdataDealBatch extends BaseBatch{
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -30,8 +30,11 @@ class ThirdPartyPriceBatch extends BaseBatch {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
"淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12","房多多":"13","我爱我家":"14",
"禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20",
"乐有家":"21","幸福里":"22","中国房价行情":"23","常州住房":"24"
]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -263,6 +266,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(third.get("csjj_esf_price")){
price_map.csjj_avg_price = third.get("csjj_esf_price")
}
if(third.get("leyoujia_esf_price")){
price_map.leyoujia_avg_price = third.get("leyoujia_esf_price")
}
if(third.get("c21_esf_price")){
price_map.c21_avg_price = third.get("c21_esf_price")
}
if(third.get("jiwu_esf_price")){
price_map.jiwu_avg_price = third.get("jiwu_esf_price")
}
if(third.get("xingfuli_esf_price")){
price_map.xingfuli_avg_price = third.get("xingfuli_esf_price")
}
if(third.get("zwfc_esf_price")){
price_map.zwfc_avg_price = third.get("zwfc_esf_price")
}
if(third.get("creprice_esf_price")){
price_map.creprice_avg_price = third.get("creprice_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -313,6 +340,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
web = "房产超市"
}else if(k=="csjj_avg_price"){
web = "城市居家"
}else if(k=="leyoujia_avg_price"){
web = "乐有家"
}else if(k=="c21_avg_price"){
web = "C21"
}else if(k=="jiwu_avg_price"){
web = "吉屋网"
}else if(k=="xingfuli_avg_price"){
web = "幸福里"
}else if(k=="creprice_avg_price"){
web = "中国房价行情"
}else if(k=="zwfc_avg_price"){
web = "常州住房"
}
ret << sourceMap.get(web)
@@ -603,6 +642,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
}
if(!dataMap.get("leyoujia_esf_price") && db.get("leyoujia_esf_price") && db.get("leyoujia_esf_price")>3000){
dataMap.put("leyoujia_esf_price", db.get("leyoujia_esf_price"))
}
if(!dataMap.get("c21_esf_price") && db.get("c21_esf_price") && db.get("c21_esf_price")>3000){
dataMap.put("c21_esf_price", db.get("c21_esf_price"))
}
if(!dataMap.get("jiwu_esf_price") && db.get("jiwu_esf_price") && db.get("jiwu_esf_price")>3000){
dataMap.put("jiwu_esf_price", db.get("jiwu_esf_price"))
}
if(!dataMap.get("xingfuli_esf_price") && db.get("xingfuli_esf_price") && db.get("xingfuli_esf_price")>3000){
dataMap.put("xingfuli_esf_price", db.get("xingfuli_esf_price"))
}
if(!dataMap.get("creprice_esf_price") && db.get("creprice_esf_price") && db.get("creprice_esf_price")>3000){
dataMap.put("creprice_esf_price", db.get("creprice_esf_price"))
}
if(!dataMap.get("zwfc_esf_price") && db.get("zwfc_esf_price") && db.get("zwfc_esf_price")>3000){
dataMap.put("zwfc_esf_price", db.get("zwfc_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -103,6 +103,8 @@ class CommonDatautils {
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -597,9 +599,9 @@ class CommonDatautils {
keys.put("region", 1);
keys.put("name", 1);
keys.put("_id", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
Iterator<DBObject> iters = cursor.iterator()
def result = [];
def result = new HashSet();
try{
while(iters.hasNext()) {
DBObject obj = iters.next()
@@ -4209,19 +4211,21 @@ class CommonDatautils {
static def findAllNewPoiDatas(city){
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
// if(map.address && map.address.contains("五角场")){
index++
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
// }
})
return rets;
}
static def findOneThirdNetWorkOffice(city, region, name){
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
def map = [:]
@@ -4965,6 +4969,20 @@ class CommonDatautils {
return regionMap
}
static def queryRangeByCat(cat){
def q = new BasicDBObject()
q.put("type",cat)
q.put("shape",new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("city_short",1)
keys.put("district_short",1)
keys.put("city",1)
keys.put("district",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList()
}
static def queryAllRegionNameByCity(){
@@ -5135,17 +5153,22 @@ class CommonDatautils {
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def distName = null
if(d.mzb_district_name){
def list = townMap.get(d.mzb_district_name)
distName = d.mzb_district_name
}else{
distName = d.district
}
if(distName){
def list = townMap.get(distName)
if(!list){
list = []
}
list << d
townMap.put(d.mzb_district_name, list)
townMap.put(distName, list)
}
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
@@ -5515,4 +5538,10 @@ class CommonDatautils {
return 0
}
static def findBusinessByCity(city){
def q = new BasicDBObject()
q.put("short_city", city)
return bk_business_coll.find(q).toList()
}
}
@@ -343,21 +343,21 @@ class CityInfoAnaBatch extends BaseBatch {
def totalDbhouseloudongCount = HOUSE_LOUDONG_CONN.count(findObj)
datas << ['', totalDistrictCount, totalLoudongCount, df.format(totalLoudongCount * 1.0/ totalDistrictCount), totalFanghaoCount, df.format(totalFanghaoCount * 1.0/ totalDistrictCount), totalDbLoudongCount, totalDbhouseloudongCount]
ExcelUtils.write(fileName, datas, resultTitle)
String id = ObjectId.get().toString()
BasicDBObject file = new BasicDBObject()
file.put("t", "upload")
file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
file.put("c_date", new Date())
file.put("_id", idGenerator(id, "o"))
file.put("data", new File(fileName).getBytes())
fsDB.save(file)
def href = DataUtil.encodePicUrl(id, "o", "upload")
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
receivers.each {receiver->
jms.sendMail(receiver, resultTitle, href)
}
new File(fileName).deleteOnExit()
// ExcelUtils.write(fileName, datas, resultTitle)
// String id = ObjectId.get().toString()
// BasicDBObject file = new BasicDBObject()
// file.put("t", "upload")
// file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
// file.put("c_date", new Date())
// file.put("_id", idGenerator(id, "o"))
// file.put("data", new File(fileName).getBytes())
// fsDB.save(file)
// def href = DataUtil.encodePicUrl(id, "o", "upload")
// href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
// receivers.each {receiver->
// jms.sendMail(receiver, resultTitle, href)
// }
// new File(fileName).deleteOnExit()
}
private String idGenerator(String url, String size) {
@@ -0,0 +1,140 @@
package com.fangjia.script.dealPrice
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
/**
* 成交同步206,按日同步
*/
@BatchOwner(Watcher.GaoJuan)
class SynchDealPriceToEs extends BaseBatch{
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static main(args) {
runBatch(new SynchDealPriceToEs(), args)
}
@Override
void process(Object args) {
run();
}
def run(){
def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
def q = new BasicDBObject()
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcursor = dealPriceConn.find(q)
println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
def ret = []
for(def o : dbcursor){
ret << o
}
save(ret)
// def batchsize = 2000
// def citys = getCityCount()
// citys.each{city,count->
// if(city instanceof String){
// for(def year = 1999;year<=2025;year++){
// def q = new BasicDBObject("city", city)
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
// println city +" "+ dbcursor.count()
// def ret = []
// for(def o : dbcursor){
// ret << o
// }
// save(ret)
// }
// }
// }
}
def getCityCount(){
BasicDBObject match = null;
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
groupFields.put("_id", new BasicDBObject("city",'$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city")) {
int count = (Integer) result.get("count");
map.put(obj.get("city"), count)
}
}
return map;
}
def save(rows){
BulkRequest request = new BulkRequest();
int batchSize = 500;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.206")
for(def map : rows){
// def map = rows.get(key)
def data_id = map._id.toString()
map.remove("_id")
request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
}
@@ -909,14 +909,18 @@ class FiledStrand {
}
static def getNumber(def input) {
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
try{
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
}
}catch(Exception e){
println input+" error"
}
return null;
}
@@ -0,0 +1,62 @@
package com.fangjia.script.gj
import groovy.json.JsonOutput
class ProxyHttpUtils {
static String proxyHost = 'tunnel2.qg.net';
static Integer proxyPort = 14186;
static String proxyUsername = '113BBC06';
static String proxyPassword = '3B2FBD05B0AF';
static String getProxyResult(targetUrl, data){
// 将 JSON 对象转换为字符串
String jsonData = new JsonOutput().toJson(data);
// 创建代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
// 创建 URL 对象
URL url = new URL(targetUrl);
// 创建连接对象
URLConnection connection = url.openConnection(proxy);
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST");
// 允许输出流
connection.setDoOutput(true);
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json");
// 发送请求参数
OutputStream outputStream = connection.getOutputStream();
outputStream.write(jsonData.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
// 读取响应
int responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
response.append(line);
}
reader.close();
return response.toString();
} else {
println "HTTP request failed with response code ${responseCode}";
}
return null;
}
}
@@ -0,0 +1,48 @@
package com.fangjia.script.gj.coresite
import java.net.HttpURLConnection
import java.net.URL
import java.io.OutputStream
// 目标服务器的URL
def targetUrlString = "http://data.zz.baidu.com/urls?site=sh.fangjia.com&token=UaKTemkcQ6I4YvSA"
// 需要POST的URL数据
def postData = "https://sh.fangjia.com/zoushi?cityName=%E4%B8%8A%E6%B5%B7&keyword=%E5%A4%A9%E5%B1%B1%E6%96%B0%E8%8B%91"
// 创建URL对象
URL url = new URL(targetUrlString)
// 打开连接
HttpURLConnection connection = (HttpURLConnection) url.openConnection()
connection.setDoOutput(true) // 表明我们将发送POST请求的输出数据
connection.setRequestMethod("POST") // 设置请求方法为POST
connection.setRequestProperty("User-Agent", "curl/7.12.1") // 设置User-Agent,尽管这在实际应用中可能并非必需
connection.setRequestProperty("Content-Type", "text/plain") // 设置内容类型,因为我们将发送纯文本数据
connection.setRequestProperty("Content-Length", Integer.toString(postData.getBytes().length)) // 设置内容长度
// 发送POST数据
//try (OutputStream os = connection.getOutputStream()) {
// byte[] input = postData.getBytes("utf-8")
// os.write(input, 0, input.length)
//}
outputStream = connection.getOutputStream();
outputStream.write(postData.getBytes());
outputStream.flush();
outputStream.close();
// 获取响应码
int responseCode = connection.getResponseCode()
println "POST Response Code :: $responseCode"
// 可选:读取响应内容
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"))
StringBuilder response = new StringBuilder()
String responseLine
while ((responseLine = reader.readLine()) != null) {
response.append(responseLine.trim())
}
println "Response Body :: ${response.toString()}"
// 关闭连接
connection.disconnect()
@@ -0,0 +1,202 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.json.JsonSlurper
import groovy.io.FileType
import org.bson.types.ObjectId
import java.util.regex.Pattern
class AddDevShapeData {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
def devAreaMap = [:]
static main(args) {
def task = new AddDevShapeData();
task.run()
// task.streetInfo()
}
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\111开发区\\'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
// dir.eachFileRecurse(FileType.FILES) { file ->
// process(file.absolutePath)
// }
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
} else {
println "The specified path is not a directory."
}
}
def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d->
// if(!d['I']){
def name = d['D']
def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A']))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
if(obj){
if(obj.operator){
d['H'] = obj.operator
}
if(obj.operate_result){
d['I'] = obj.operate_result
}
if(obj.m_date){
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
}
}
// }
devAreaMap.each {k,v->
if(name.contains(k)){
d['M'] = v.area_info
d['N'] = v.count
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
}
}
})
}
def process(file){
def retMap = getNewLocations(file)
if(retMap && retMap.gaode.size()>0){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+retMap.name+'.*$', Pattern.CASE_INSENSITIVE)))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
if(obj){
for(def i=0;i<retMap.gaode.size();i++){
def udata = new BasicDBObject()
def resultMap = [:]
def bd_resultMap = [:]
resultMap.type = "polygon"
resultMap.coordinates = [retMap.gaode[i]]
bd_resultMap.type = "polygon"
bd_resultMap.coordinates = [retMap.baidu[i]]
if(i>0){
obj.removeField("_id")
obj.put("create_time", new Date())
obj.put("block", retMap.name+"-地块"+i)
obj.put("shape", resultMap)
obj.put("m_date", new Date())
obj.put("bd_shape",bd_resultMap)
obj.put("operator", "外包")
obj.put( "operate_result" , "已处理")
DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{
udata.put("shape", resultMap)
udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "已处理")
udata.put("operator", "外包")
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
}
}
}
}
def getNewLocations(file){
def jsonObj = readJson(file)
def features = jsonObj.features
def gaode_locations = []
def baidu_locations = []
def name = null
def areaMap = [:]
if(features && features.size()==1){
name = features[0].properties.Name
def coordinates = features[0].geometry.coordinates
def index = 0
println name +" 有 "+coordinates.size()+" 块"
areaMap.count = coordinates.size()
def areas = []
if(name.contains("泉州经济技术开发区")){
for(def pointList : coordinates){
index ++
def gaode_coordinates = []
def bd_coordinates = []
def points = null
if(pointList.size()>1){
points = pointList
}else{
points = pointList[0]
}
double area = com.fangjia.script.util.GeoHelper.calculateArea(points)
println name +" 地块"+index + "围栏面积是: ${area} 平方公里"
areas << area
def locationstr = ""
for(def arrs : points){
def gaode_map = GeoUtils.gps84ToGcj02(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
gaode_coordinates << [gaode_map.get("lng"), gaode_map.get("lat")]
//此高德转百度 有偏差
def bd_map = GeoUtils.gcj02ToBd09(gaode_map.get("lat"), gaode_map.get("lng"))
// bd_coordinates << [bd_map.get("lng"), bd_map.get("lat")]
// 高德转百度 通过接口有次数限制
// def baidu = BaiDuApiUtils.geoconv(gaode_map.get("lng"), gaode_map.get("lat"), 3)
// bd_coordinates << [baidu.x, baidu.y]
locationstr = arrs[0].toString()+","+arrs[1].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()), 6)
if(bdpoint){
bd_coordinates << bdpoint
}
}
//整个一串一起传超过4029会报错
// bd_coordinates = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()-1), 6)
gaode_locations << gaode_coordinates
if(bd_coordinates){
baidu_locations << bd_coordinates
}else{
println name +" 转换有问题"
}
}
areaMap.area_info = areas.join("\n")
}
}else{
println file +" 有多个"
}
devAreaMap.put(name, areaMap)
def retMap = [:]
retMap.name = name
retMap.gaode = gaode_locations
retMap.baidu = baidu_locations
return retMap
}
def readJson(def file){
def jsonSlurper = new JsonSlurper()
def jsonFile = new File(file)
def jsonText = jsonFile.text
def jsonObj = jsonSlurper.parseText(jsonText)
return jsonObj
}
def streetInfo(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx", {d->
def count = dev_mapping_CONN.count(new BasicDBObject("name", d['E']).append("type", "乡镇街道"))
d['L'] = count
})
}
}
@@ -0,0 +1,448 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.json.JsonSlurper
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//补充库中没有详细地址的小区
class AddDistrictAddress extends BaseBatch{
// def fullCityMap = CommonDatautils.queryFullNameCity()
// def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
//
// DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
// public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
//
//
// public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
//
// def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
// DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
//
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new AddDistrictAddress(), args)
}
@Override
public void process(Object args) {
// addNoAddressDist() //为不精确的地址通过地图补充地址
// mapOtherAddress() //有精确地址的寻找其它地址
// addressSelect()
addrUpdateToDB()
}
def addNoAddressDist(){
// def citys = CommonDatautils.findAllCity()
// def index = 0
// def allrets = []
// for(def c : citys){
// index ++
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
//
// def rets = []
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
// for(def d : noRoadInfoDists){
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
// }
// allrets.addAll(rets)
// if(allrets.size()>50000){
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
// allrets.clear()
// }
// }
updateDistAddr()
}
def mapOtherAddress(){
def citys = CommonDatautils.findAllCity()
def index = 0
def allrets = []
for(def c : citys){
index ++
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address", "addr_alias"])
def radInfoDists = haveRoadInfoDistricts(districts)
println c +" "+index +" 原始小区量:"+districts.size() +" 有详细地址量:"+radInfoDists.size()
radInfoDists = baiduMatch(c, radInfoDists)
radInfoDists = gaodeMatch(c, radInfoDists)
def rets = []
rets << ["id", "城市", "区域", "名称", "地址","地址别名", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
for(def d : radInfoDists){
def addr_alias_str = '';
if(d.addr_alias){
addr_alias_str = d.addr_alias.join(" ")
}
rets << [d._id.toString(), d.city, d.region, d.name, d.address, addr_alias_str,d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
}
allrets.addAll(rets)
if(allrets.size()>50000){
ExcelUtils.write("D:\\数据\\小区\\小区精确地址补充"+index+".xlsx", allrets, "补充地址")
allrets.clear()
}
}
}
def addressSelect(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-all-2.xlsx",{d->
if(d['G']){
d['O'] = AddressUtil.formatAddress(d['B'], d['G'])
}
if(d['I']){
d['P'] = AddressUtil.formatAddress(d['B'], d['I'])
}
})
}
def addrUpdateToDB(){
def updateDist = 0
def addAddrCount = 0
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-入库-2.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def oldAddrAliasCount = 0
if(dist.addr_alias){
oldAddrAliasCount = dist.addr_alias.size()
}
def new_addr_alias = null
if(d['K'].contains("新增百度地址") || d['K'].contains("双地图新增")){
new_addr_alias = getNewAddressAlias(d['L'], dist.address, dist.addr_alias)
}
if(d['K'].contains("新增高德地址") || d['K'].contains("双地图新增")){
new_addr_alias = getNewAddressAlias(d['M'], dist.address, dist.addr_alias)
}
if(new_addr_alias){
updateDist++
println updateDist
addAddrCount = addAddrCount + new_addr_alias.size()-oldAddrAliasCount
def udata = new BasicDBObject()
udata.put("addr_alias", new_addr_alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
}
}
})
println "修改小区数:"+updateDist
println "新增地址数:"+addAddrCount
}
def getNewAddressAlias(new_addr, old_addr, dist_addr_alias){
def arrs = new_addr.split(";")
def addr_alias = []
if(dist_addr_alias){
addr_alias = dist_addr_alias
}
for(def address : arrs){
if(old_addr!=address && !addr_alias.contains(address)){
addr_alias << address
}
}
return addr_alias
}
def updateDistAddr(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("address", d['J'])
def addr_alias = dist.addr_alias
if(!dist.addr_alias){
addr_alias = []
}
if(d['K'] && !addr_alias.contains(d['K'])){
addr_alias << d['K']
}
if(addr_alias){
udata.put("addr_alias", addr_alias)
}
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
d['L'] = "已更新"
}
})
}
def noDetailRoadInfoDistricts(districts){
def rets = []
for(def d : districts){
if(d.address){
def flag = DataHandleUtils.isStandardRoad(d.address)
if(!flag){
rets << d
}
}else{
rets << d
}
}
return rets;
}
def haveRoadInfoDistricts(districts){
def rets = []
for(def d : districts){
if(d.address){
def flag = DataHandleUtils.isStandardRoad(d.address)
if(flag){
rets << d
}
}
}
return rets;
}
def baiduMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = baiduProcess(d, fullCityName)
if(retMap){
d.baidu_address = retMap.new_addr
d.baidu_old_addr = retMap.old_addr
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def baiduProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
for(def name : alias){
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
if(temp_address){
retMap.old_addr = temp_address
}
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
retMap.new_addr = nineAddrSplit(fullCityName, temp_address)
continue
}
}
return retMap
}
def nineAddrSplit(city, addr){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
def address = ""
if (jsonObject) {
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
address = jsonObject.get("street")
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
address = address + jsonObject.get("streetNumber")
}
}
return address
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def dbcursor = BAIDU_POI_COLL.find(q)
def address = null
for(def d : dbcursor){
address = d.addr
}
return address;
}
def gaodeMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = gaodeProcess(d, fullCityName)
if(retMap){
d.gaode_addr = retMap.new_addr
d.gaode_old_addr = retMap.old_addr
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def gaodeProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
def address = null
for(def name : alias){
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
if(temp_addr){
retMap.old_addr = temp_addr
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
retMap.new_addr = nineAddrSplit(fullCityName, temp_addr)
continue
}
}
}
return retMap
}
def findGaoDeDatas2(city, region, name){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
def body = """
{
"_source": {
"includes": ["cat", "address", "id","name"]
},
"query": {
"bool": {
"must": [
{
"term": {
"city" : "${city}"
}
},
{
"term": {
"district" : "${region}"
}
},
{
"match": {
"name": {"query": "${name}","operator": "and"}
}
}
// ,{
// "match": {
// "small_type": "住宅"
// }
// }
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def address = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
def d = hits[0]
if(d._source.cat.contains("住宅")){
def source = d._source
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
if(newName == name){
address = source.address
}
}
}
}
return address
}
}
@@ -57,7 +57,7 @@ class AddDistrictShapeBatch extends BaseBatch{
//
// }
checkRangeCount()
// checkRangeCount()
// synch100()
}
@@ -0,0 +1,98 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ExcelUtils
class AddrAlaisProcess {
static main(args) {
def task = new AddrAlaisProcess()
// task.run()
task.nineApi()
}
def nineApi(){
def fileName = "/home/goojia/gj/district/addr_alias.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-测试.xlsx"
}
ExcelUtils.multiThreadingModify(fileName, {d->
def addrMap = nineAddrSplit(d['B'], d['E'])
def addrList = []
if(addrMap && addrMap.address){
d['G'] = addrMap.city
d['H'] = addrMap.address
addrList << addrMap.address
}
if(d['F']){
def addrs = d['F'].split(",")
def addr_alias = []
for(def addr : addrs){
def map = nineAddrSplit(d['B'], addr)
if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
if(!addrList.contains(map.address)){
addrList << map.address
addr_alias << map.address
}
}
}
d['I'] = addr_alias.join(",")
}
})
}
def nineAddrSplit(city, addr){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
def address = ""
def map = [:]
if (jsonObject) {
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
map.province = jsonObject.get("province")
}else{
map.province = ""
}
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
map.city = jsonObject.get("city")
}else{
map.city = ""
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
map.district = jsonObject.get("district")
}else{
map.district = ""
}
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
address = jsonObject.get("street")
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
address = address + jsonObject.get("streetNumber")
}
map.address = address
}
return map
}
def run(){
def citys = CommonDatautils.findAllCity()
def rows = []
citys.each{c->
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","address", "addr_alias"])
println c +" "+districts.size()
def num = 0
for(def d : districts){
num++
def row = [num, d.city, d.region, d.name, d.address]
if(d.addr_alias){
row << d.addr_alias.join(",")
}
rows << row
}
}
// ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区")
FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.bat", null, "utf-8", false, "#","");
}
}
@@ -35,22 +35,22 @@ class AutoAddDistrictBatch extends BaseBatch{
@Override
void process(Object args) {
// //查找第三方匹配没有匹配上的小区
// def citys = CommonDatautils.findAllCity()
// def list = []
// list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
// for(def c : citys){
// def noMatchDatas = findNoMatchResult(c)
// println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
// def rets = checkDatas(noMatchDatas.keySet())
// list.addAll(rets)
// }
//查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
def rets = checkDatas(noMatchDatas.keySet())
list.addAll(rets)
}
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\Êý¾Ý\\add_dist_info_1.xlsx"
}
// ExcelUtils.write(fileName, list, "待入库数据")
ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
@@ -0,0 +1,268 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class BusinessQuarterProcess {
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def fullCityMap = CommonDatautils.queryFullNameCity()
DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020)
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new BusinessQuarterProcess()
task.run()
}
def run(){
// synchBusinessData()
// updateBusinessQuarter()
// addRegion()
synchBusinessToEs()
}
def addRegion(){
def list = CommonDatautils.queryRangeByCat("区域")
println list.size()
def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2")))
for(def d : dbcursor){
if(d.district){
continue
}
for(def regionShape : list){
if(d.short_city!=regionShape.city_short){
continue
}
def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates))
if(flag){
def udata = new BasicDBObject()
udata.put("district", regionShape.mzb_district_name)
udata.put("short_district", regionShape.district_short)
bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
break
}
}
}
}
def synchBusinessData(){
def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList()
synchMongoProcess(datas)
}
def synchMongoProcess(datas){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
createData(d)
println d.city +" "+totalCount.get() +"/"+datas.size()
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def createData(d){
def row = new BasicDBObject()
row.put("_id", d._id)
row.put("short_province", d.province)
def short_city = RegionData.formatCity(d.city)
row.put("short_city", short_city)
row.put("name", d.name)
row.put("bd_location", ["lon":DataHandleUtils.getDoubleRound(6,d.longitude), "lat":DataHandleUtils.getDoubleRound(6,d.latitude)])
def data = fullCityMap.get(short_city)
if(data){
row.put("province", data.province)
row.put("city", data.city)
}else{
println d.city + " 没找到简称"
}
def gdPoints = FangjiaClientUtils.convert(d.longitude+","+d.latitude, 3)
if(gdPoints){
row.put("location", ["lon":DataHandleUtils.getDoubleRound(6,gdPoints[0]), "lat":DataHandleUtils.getDoubleRound(6,gdPoints[1])])
}
def border = d.border
def bd_shapes = []
def gd_shapes = []
if(border){
def arrs = border.split(";")
if(border.contains("|")){
println d._id.toString()+" 可能含多个围栏"
}
for(def i=0;i<arrs.size();i++){
def str = arrs[i]
def arr = str.split(",")
def bd_lng = DataHandleUtils.getDoubleRound(6, arr[0].toString().toDouble())
def bd_lat = DataHandleUtils.getDoubleRound(6, arr[1].toString().toDouble())
if(!bd_shapes.contains([bd_lng, bd_lat]) ){
bd_shapes << [bd_lng, bd_lat]
}
if(bd_shapes.contains([bd_lng, bd_lat]) && i+1==arrs.size() && bd_shapes[0]==[bd_lng, bd_lat]){
bd_shapes << [bd_lng, bd_lat]
}
def tempGdPoints = FangjiaClientUtils.convert(bd_lng+","+bd_lat, 3)
if(tempGdPoints){
def gdlist = [DataHandleUtils.getDoubleRound(6,tempGdPoints[0]), DataHandleUtils.getDoubleRound(6,tempGdPoints[1])]
if(!gd_shapes.contains(gdlist)) {
gd_shapes << gdlist
}
if(gd_shapes.contains(gdlist) && i+1==arrs.size() && gd_shapes[0]==gdlist){
gd_shapes << gdlist
}
}
}
if(bd_shapes[0] != bd_shapes[bd_shapes.size()-1]){
bd_shapes << bd_shapes[0]
gd_shapes << gd_shapes[0]
}
}
def gdMap = [:]
gdMap.put("type" , "polygon")
gdMap.put("coordinates", [gd_shapes])
row.put("shape", gdMap)
def bdMap = [:]
bdMap.put("type" , "polygon")
bdMap.put("coordinates", [bd_shapes])
row.put("bd_shape", bdMap)
row.put("c_date", new Date())
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
}
def updateBusinessQuarter(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
if(["上海","北京"].contains(c)){
continue
}
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
businessProcess(c, districts, cityBusinessList)
}
}
def businessProcess(c,districts, cityBusinessList){
def updateIndex = 0
for(def d : districts){
if(!d.business_quarter){
def businessQuarter = findBusinessByRange(d, cityBusinessList)
if(businessQuarter){
updateIndex ++
def udata = new BasicDBObject()
udata.put("business_quarter", businessQuarter)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
println c +" "+ districts.size() +" "+updateIndex
}
def findBusinessByRange(d, cityBusinessList){
def name= null
for(def shapeObj : cityBusinessList){
if(d.lng){
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(shapeObj.shape.coordinates))
if(flag){
name = shapeObj.name
break
}
}
}
return name
}
def synchBusinessToEs(){
def dbcursor = bk_business_coll.find(new BasicDBObject())
def rows = []
def index = 0
for(def d : dbcursor){
index++
if(index<5000){
continue
}
// rows << d
// if(rows.size()>=100){
// updateByEs(rows)
// rows.clear()
// }
updateByEs(d, index)
}
}
def updateByEs(row, index){
BulkRequest request = new BulkRequest();
def client = ElasticUtils89.getClient("192.168.0.89")
int processedCount = 0;
def id = row.get("_id").toString()
def doc = [:]
doc.put("province",row.province)
doc.put("short_province",row.short_province)
doc.put("city",row.city)
doc.put("short_city",row.short_city)
doc.put("district",row.district)
doc.put("short_district",row.short_district)
doc.put("name", row.name)
doc.put("location", row.location)
doc.put("bd_location", row.bd_location)
doc.put("shape", row.shape)
doc.put("bd_shape", row.bd_shape)
request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON));
if (request.numberOfActions() >= 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
// println "Processed " + processedCount + " documents"
}
}
}
}
@@ -72,7 +72,7 @@ class DevProcess {
// println DateUtils.add("2024-7-24", 7);
// task.addKfc()
// task.devToPark() //开发区下辖园区
// task.devToPark() //开发区下辖园
// task.checkTown()
// task.repeatSave()
@@ -84,6 +84,7 @@ class DevProcess {
// task.addQiDevResult() //获取齐梁数据结果
// task.updateDevShape()//修改开发区结构
}
@@ -439,30 +440,33 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
if(!d['B'] && d['A']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
if(!d['E'] && d['D']){
def q = new BasicDBObject()
// q.put("name", d['A'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['A']]))
def name = d['A'].replace('经济技术开发区','经济开发区')
// q.put("name", d['D'])
q.put("city",d['C'])
// q.put('alias', new BasicDBObject('$in',[d['D']]))
def name = d['D'] //.replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['I'] = "库中有开发区-相似2"
d['G'] = "相似"
// d['K'] = obj.name
// d['C'] = obj._id.toString()
d['B'] = obj.name
// if(obj.name == d['A']){
// d['E'] = "相同"
d['E'] = obj.name
// if(obj.name == d['D']){
// d['G'] = "相同"
// }else{
// d['E'] = "包含"
// d['G'] = "包含"
// }
if(obj.shape){
d['F'] = "是"
}
// d['E'] = "别名"
// d['F'] = obj.name
}
}
},1);
},0);
}
def calAddrDistince(){
@@ -0,0 +1,113 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.ExcelUtil
import com.fangjia.script.gj.ProxyHttpUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
class DevQlCheck {
//http://146.56.217.44:8088/uams/index
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static main(args) {
def task = new DevQlCheck()
task.run()
}
def run(){
def devName = "福建龙海经济开发区"
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",devName))
def pois = readCSV("C:\\Users\\gaojuan\\Downloads\\漳州市POI.csv")
def devPois = []
devPois << ["省", "城市", "区域", "名称", "地址", "类型", "经度", "纬度", "齐梁开发区"]
for(def poi : pois){
def flag = InsidePolygon.isInArea(poi.lng.toString().toDouble(), poi.lat.toString().toDouble(), InsidePolygon.createPoint4(obj.shape.coordinates))
if(flag){
devPois << [poi.province, poi.city, poi.district, poi.name, poi.address, poi.cat, poi.lng, poi.lat]
}
}
println "共:"+devPois.size()+" 个"
ExcelUtils.write("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx", devPois, "齐梁开发区")
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx",{d->
def address = d['B']+d['C']+d['D']
// if(d['E']){
// address = address + d['E']
// }
Thread.sleep(3000)
def s = getQlResult(address)
d['I'] = s
index ++
println index+" "+ s
})
}
def getQlResult(address){
def params = [
'address': address,
'type': '1' //1:对公 0:对私
]
try{
def str = ProxyHttpUtils.getProxyResult(targetUrl, params);
println address +" "+ str
JSONObject result = JSONObject.fromObject(str)
if(result && result.code==20000){
def data = result.data
if(data){
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
return s
}
}
}
}catch(Exception e){
println address +" "+e.getMessage()
}
return null
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
try{
o.put("province", arrs[0])
o.put("city", arrs[1])
o.put("district", arrs[2])
o.put("name", arrs[3])
o.put("township", arrs[4])
o.put("address", arrs[5])
o.put("cat", arrs[6])
o.put("lng", arrs[7])
o.put("lat", arrs[8])
o.put("data_source", arrs[9])
rets << o
}catch(Exception e){
println filename+" 文件异常"
}
}
}
return rets;
}
}
@@ -32,7 +32,7 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
//
@@ -167,7 +167,7 @@ class DevUpdateTestBatch {
println d['A'] +" 不存在"
}
}
},0)
},1)
}
@@ -1,12 +1,14 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -28,14 +30,24 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
task.addOldMatch2()
// task.addOldMatch2()
// task.runDb()
// task.hainanMatch()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
// task.runMatch()
// task.checkDist()
task.addPrice()
}
def addPrice(){
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
}
def addOldMatch(){
@@ -94,6 +106,46 @@ class GXDdistrictMatch {
}
def hainanMatch(){
def fileName = "D:\\数据\\小区\\海南未匹配地址-1.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['D'] && d['F'] ){//
// def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['I'])
// if(jsonObject){
//// if(jsonObject.get("district") && jsonObject.get("district")!='null'){
//// d['AO'] = jsonObject.get("district")
//// }
// if(jsonObject.get("province") && jsonObject.get("province")!="null"){
// d['AP'] = jsonObject.get("province")
// }
// if(jsonObject.get("city") && jsonObject.get("city")!="null"){
// d['AI'] = jsonObject.get("city")
// }
// }
def formatCity = RegionData.formatCity(d['F'])
def fullCityObj = fullCityMap.get(formatCity)
if(fullCityObj){
// def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
def obj = findDatasByCityAndName(formatCity, d['D'].replaceAll('[·\\(\\)()]|·', ''))
if(obj){
d['U'] = "楼盘抓取库"
d['V'] = obj.city
d['W'] = obj.region
d['X'] = obj.name
d['Y'] = obj.property_type
d['Z'] = obj.address
def loc = obj.location
if(loc){
d['AA'] = loc.x
d['AB'] = loc.y
}
}
}
}
})
}
def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
@@ -238,43 +290,41 @@ class GXDdistrictMatch {
def saveToDB(){
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
def fileName = "D:\\数据\\小区\\海南新增小区.xlsx"
ExcelUtils.modify(fileName, { d ->
if(d['A']){
if(d['A'] && d['C']){
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def alias = []
if(d['G'] && name!=d['G']){
alias << d['G']
}
// def alias = []
// if(d['G'] && name!=d['G']){
// alias << d['G']
// }
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
// alias << d['AI']
// }
if(alias){
udata.put("alias", alias)
}
// if(alias){
// udata.put("alias", alias)
// }
def addr = AddressUtil.formatAddress(d['A'], d['D'])
def addr = AddressUtil.formatAddress(d['A'], d['E'])
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("status", 0)
udata.put("name_cat", 0)
udata.put("cat", "district")
udata.put("username", "高娟")
// udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
// if(d['AH']){
// type = d['AH']
// }else{
// type = d['Y']
// }
if(d['D']){
type = d['D']
}
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
@@ -282,26 +332,50 @@ class GXDdistrictMatch {
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
udata.put("estate_type2", estateType2)
if(d['F']){
Long lng = d['F']*10000000000
udata.put("lng", lng)
}
if(d['E']){
Long lat = d['E'] * 10000000000
Long lat = d['G'] * 10000000000
udata.put("lat", lat)
}
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
def opt = "add"
if(s && s=='成功'){
saveOptLog(udata.city, udata.region, udata.name, opt)
udata.put("lng", lng)
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(d['F'].toString().toDouble(), d['G'].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
if(lng2){
Long lng2_new = lng2 *10000000000L
Long lat2_new = lat2 *10000000000L
udata.put("lng2", lng2_new)
udata.put("lat2", lat2_new)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map",markMap)
// udata.put("is_new", 0)
DbUtils.seaweed.save(udata)
saveOptLog(udata.city, udata.region, udata.name, "add")
// def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println name
// def opt = "add"
// if(s && s=='成功'){
// saveOptLog(udata.city, udata.region, udata.name, opt)
// }
}
})
}
def checkDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\海南未匹配地址-1.xlsx", { d ->
if(d['V'] && d['W'] && d['X']){
matching(d['V'], d['W'], d, d['X'], null, null)
}
})
}
def match(city, region, d){
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
@@ -391,7 +465,7 @@ class GXDdistrictMatch {
if(rets && rets.size()>=1){
d = setRowValue(rets[0], d, "名称在地址中"+str)
}else{
if (d['D']) {
if (d['D'] && alias) {
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
if (rets) {
if (rets.size() >= 1) {
@@ -449,14 +523,14 @@ class GXDdistrictMatch {
def setRowValue(dbo, d, remark){
if (dbo) {
d['V'] = dbo.city
d['W'] = dbo.region
d['X'] = dbo.name
d['Y'] = dbo.estate_type2
d['Z'] = dbo.status
d['AA'] = dbo.name_cat
d['U'] = remark
d['AH'] = dbo.address
// d['V'] = dbo.city
// d['W'] = dbo.region
// d['X'] = dbo.name
// d['Y'] = dbo.estate_type2
d['AK'] = dbo.status
// d['AA'] = dbo.name_cat
d['AJ'] = remark
// d['AH'] = dbo.address
}
return d;
}
@@ -510,6 +584,33 @@ class GXDdistrictMatch {
return null;
}
def findDatasByCityAndName(def city, def name){
def sql = "select * from community where city='${city}' and name='${name}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
println d.property_type
})
if(rets){
def obj = null
for(def r : rets){
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
obj = r
break
}
}
if(obj){
return obj;
}else{
return rets[0];
}
}
return null;
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -39,12 +46,13 @@ class NewGaoDePoiBatch extends BaseBatch{
@Override
void process(Object args) {
run()
// run()
oldPoiMagre()
}
def run(){
if (CommonDatautils.isWindow()) {
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
TAGS = getAllPoiType("D:\\数据\\数据建设\\Poi\\POI分类白名单.xlsx")
}else{
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
}
@@ -145,6 +153,7 @@ class NewGaoDePoiBatch extends BaseBatch{
saveObj.data_id = map.map_id
saveObj.data_source = "¸ßµÂµØÍ¼"
saveObj.data_source_year = "2024-11"
saveObj.parent_id = map.parent_id
if(saveObj.cat && TAGS.contains(saveObj.cat)){
saveObj.status = 0
@@ -314,7 +323,7 @@ class NewGaoDePoiBatch extends BaseBatch{
rows.each{map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
request.add(new IndexRequest("map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
// ÅúÁ¿Ìá½»Êý¾Ý
if (request.numberOfActions() >= batchSize) {
@@ -346,4 +355,177 @@ class NewGaoDePoiBatch extends BaseBatch{
}
def oldPoiMagre(){
def cityList = CommonDatautils.findFullNameCitys().reverse()
def index = 0
for(def cityObj : cityList){
index ++
if(index <291){
continue
}
def fileName = "/home/goojia/gj/poi/old_poi/merge_poi/${cityObj.city}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
}
println fileName
File file=new File(fileName);
if(file.exists()){
def results = readCSV(fileName)
println "运行城市:"+cityObj.province+" " +cityObj.city+" "+results.size()
if(results){
def megerMap = poiRepeat(cityObj.city, results)
println "待保存:" + cityObj.city +" "+megerMap.size()
save(megerMap.values())
}
}else{
println fileName +" 不存在"
}
}
}
def poiRepeat(city, oldPois){
def rets = new ConcurrentHashMap()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(oldPois.size())
def townMap = CommonDatautils.queryTownByCity(city)
def esMap = findEsPoiByCity(city)
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
for(def i=0;i<oldPois.size();i++){
final def p = oldPois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def nameMap = esMap.get(p.district)
def nameKeys = nameMap.keySet()
def key = p.name.replace("-","").replace("·","")
if(!nameKeys.contains(key)){
def simlarObj = null
if(esMap){
simlarObj = findSimlarPoi(p, esMap)
}
if(!simlarObj){
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
if(!rets.containsKey(newKey)){
p.data_source_year = "2023-08"
rets.put(newKey, p)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findSimlarPoi(ret, esMap){
def simlarObj = null
def region = ""
if(ret.district){
region = ret.district
}
def regionMap = esMap.get(region)
if(regionMap){
def keys = regionMap.keySet()
for(def key : keys){
def esValue = regionMap.get(key)
if(esValue.township && ret.township){
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
if(ret.location && ret.location.lon){
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
if(distance <100){
simlarObj = esValue
println esValue.name +" "+ret.name+" "+distance
break;
}
}else{
println ret.data_id +" 无经纬度信息"
}
}
}
}
}else{
println "无数据"
}
return simlarObj
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def map = new HashMap()
ElasticUtils89.executeQuery("192.168.0.89", "map_poi_2024", queryBuilder, { String id, Map d->
def district = "嘉峪关市"
if(d.district){
district = d.district
}
def regionMap = map.get(district)
if(!regionMap){
regionMap = new HashMap()
}
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
regionMap.put(d.name.replace("-","").replace("·",""), o)
// map.put(district, regionMap)
map.put(district, regionMap)
})
return map
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("province",arrs[0])
o.put("city" , arrs[1])
o.put("district",arrs[2])
o.put("name",arrs[3])
o.put("township",arrs[4])
o.put("address",arrs[5])
o.put("cat",arrs[6])
o.put("type",arrs[7])
o.put("secondary_type",arrs[8])
o.put("small_type" ,arrs[9])
if(arrs[10]){
o.put("data_id",arrs[10])
}else{
o.put("data_id",arrs[11])
}
o.put("spell_name",arrs[12])
def location = [:]
if(arrs[13]){
location.lon = arrs[13].toString().toDouble()
location.lat = arrs[14].toString().toDouble()
}
o.put("location",location)
o.put("data_source",arrs[15])
o.put("status",arrs[16])
o.put("version",arrs[17])
rets << o
}
}
return rets;
}
}
@@ -6,7 +6,7 @@ import com.fangjia.script.util.ExcelUtils
class PoiNineSplitBatch{
static main(args) {
public static void main(args) {
def task = new PoiNineSplitBatch()
task.run()
}
@@ -52,9 +52,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
def index = 0
for(def c : citys){
index ++
if(index < 206){
continue
}
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
matchByCity(c, noMatchDatas)
@@ -123,9 +120,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
def index = 0
for(def c : citys){
index ++
if(index < 70){
continue
}
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
def addCount = addDatasToDB(crawlDatas)
println c + " ÐÂÔö£º"+ addCount
@@ -3,15 +3,29 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -22,6 +36,7 @@ class TownStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def updateList = []
def addAliasList = []
@@ -38,7 +53,9 @@ class TownStandardBatch extends BaseBatch{
def TownStandard(){
// townStandardByCat("office")
townStandardByPoi()
// townStandardByPoi()
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// updateTownData()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
@@ -69,11 +86,15 @@ class TownStandardBatch extends BaseBatch{
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
//浙江,江苏,安徽
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
def runProvinces = ["福建省","浙江省","江苏省","安徽省","山东省","甘肃省"]
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index++
def data = fullCityMap.get(city)
// if(runProvinces.contains(data.province)){
// continue
// }
def fullCityName = null
if(data){
fullCityName = data.city
@@ -95,11 +116,16 @@ class TownStandardBatch extends BaseBatch{
def rets = readPoiCsv(fileName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data)
}else{
// haveStreetProcess(city, set, cat)
}
}
if(updateList.size()>50000){
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-${index}.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名-${index}.xlsx", addAliasList, "数据")
updateList.clear()
}
}else{
println fileName+" 文件没找到"
}
@@ -107,8 +133,7 @@ class TownStandardBatch extends BaseBatch{
println "没找到相应的全称:"+city
}
}
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-最后.xlsx", updateList, "数据")
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
@@ -168,7 +193,7 @@ class TownStandardBatch extends BaseBatch{
}
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data){
def results = []
if(fullCityName){
for(def d : rets){
@@ -221,21 +246,21 @@ class TownStandardBatch extends BaseBatch{
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.province, d.city, d.district, d.name,d.township, d.status, updateTown]
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
// results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
results << m
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.province, d.city, d.district, d.name, d.township, d.status, ""]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
}
}
@@ -244,10 +269,10 @@ class TownStandardBatch extends BaseBatch{
}
}
// if(results.size()>0){
// updateByEs(results)
// results.clear()
// }
if(results.size()>0){
updateByEs(results)
results.clear()
}
println fullCityName +" 处理完成"
}
@@ -322,18 +347,334 @@ class TownStandardBatch extends BaseBatch{
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
try{
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}catch(Exception e){
println arrs
}
}
}
return rets;
}
def readCSV2(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
def o = new BasicDBObject()
try{
o.put("id", arrs[1])
o.put("province", arrs[2])
o.put("city", arrs[3])
o.put("district", arrs[4])
o.put("lng", arrs[5])
o.put("lat", arrs[6])
if(arrs.size()<=7){
o.put("data_source","")
}else{
o.put("data_source", arrs[7])
}
rets << o
}catch(Exception e){
println arrs
}
}
index++
}
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
if(arrs.size()>=9 && arrs[8]){
o.put("district", arrs[8])
}
if(arrs.size()>=10 && arrs[9]){
o.put("way", arrs[9])
}
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def addTownByTownShape(){
def i = 0
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
while(i<50){
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
def rets = readCSV2(fileName)
println i +" "+ rets.size()
def rows = addTownshipBatch(rets)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
println fileName +" 运行完成"
}else{
println fileName +" 不存在"
}
i++
}
}
def addTownshipBatch(def rets){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
if (rMap) {
def townMap = [:]//CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
// def set = rMap.keySet()
println totalCount.get() +" "+r.city + " "
// if (set.size() >= 1) {
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
// }
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return results
}
def addTown(saveObj, townMap, location) {
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
// def towns = townMap.get(saveObj.district)
// if (towns) {
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.town_source = "同区域"
// saveObj.strand_town = "是"
// break
// }
// }
// }
// }
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
// if(!saveObj.township){
// def townNames = townMap.keySet()
// for(def townName : townNames){
// towns = townMap.get(townName)
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.new_district = town.mzb_district_name
// saveObj.town_source = "忽略区域"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// break
// }
// }
// }
// if(saveObj.township){
// break;
// }
// }
// }
//通过调取地图api接口获取乡镇街道
if(!saveObj.township){
def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
if(gaodeMap && gaodeMap.township){
saveObj.township = gaodeMap.township
saveObj.new_district = gaodeMap.district
saveObj.town_source = "高德API"
saveObj.strand_town = ""
// if(townMap.size()>=1){
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// }
}
}
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
// if(addressComponent.town){
// saveObj.township = addressComponent.town
// saveObj.new_district = addressComponent.district
// saveObj.town_source = "百度API"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// }
// }
// }
// }
return saveObj
}
def isStandTown(townMap, district, town){
def towns = townMap.get(district)
def flag = "非"
if(towns){
for(def t : towns){
if(t.street == town){
flag = "是"
break
}
}
}
return flag
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def stop = false
def count = 0
def jsonObject = null
// while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
city = addressComponent.city
format_addr = jsonObject.regeocode.formatted_address
}
}
}
// if(jsonObject.status==0 || count>3){
// stop = true
// println "高德调用超3次"
// }else{
// count ++
// Thread.sleep(3000)
// print "."
// }
// }
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
def updateTownData(){
def i = 0
while(i<50) {
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv"
}
File file = new File(fileName);
if (file.exists()) {
def rets = readCSV3(fileName)
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
// if(rets){
// updateToEs(rets)
// }
}
i++
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.containsKey("way")){
map.remove("way")
}
request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
}
@@ -38,16 +38,16 @@ class TownshipStrandUpdate {
static main(args) {
def task = new TownshipStrandUpdate()
// task.process()
// task.updateTown()
task.process()
task.updateTown()
// task.stat()
task.updateDiffRegionTown()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// println new Date().getTime()
// task.updateSpecCityTown()
}
@@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.text.SimpleDateFormat
import java.util.concurrent.CopyOnWriteArrayList
@@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
@@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
final static def EMAILS = [
"gaojuan@fangjia.com"
@@ -68,6 +71,38 @@ class DistrictFieldAddBatch extends BaseBatch{
static main(args) {
runBatch(new DistrictFieldAddBatch(), args)
// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
//// if(dist){
//// d['L'] = dist.loudong_count
//// d['M'] = dist.household_count
//// d['Q'] = dist.estate_type2
//// }
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
// if(!list && dist){
// def alias = dist.alias
// for(def name : alias){
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
// if(list){
// d['Z'] = "别名"
// }
// }
// }else{
// d['Z'] = "名称"
// }
// if(list){
// def obj = list[0]
// if(obj.loudong_cnt){
// d['W'] = obj.loudong_cnt
// }
//
// if(obj.households){
// d['X'] = obj.households
// }
// d['Y'] = obj.url
// }
//
// })
}
@Override
@@ -83,18 +118,13 @@ class DistrictFieldAddBatch extends BaseBatch{
headers << fieldsNameMap.get(it)
}
headers << "抓取量"
headers << "匹配量"
headers << "0.9分以上量"
headers << "匹配率"
headers << "0.9以上匹配率"
// analysRets << headers
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
if(totalCount){
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表
}
}
println analysRets.toString()
@@ -107,8 +137,10 @@ class DistrictFieldAddBatch extends BaseBatch{
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
def matchRets1 = matchApi(esRets)
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
@@ -140,6 +172,65 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def addFieldFromMatchMapping(c, totalCount, analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findMatchSameDatas(c, esWebSource)
println esWebSource +" 新老算法匹配相同量:"+esRets.size()
def updateRets = processFieldsFromMatch(c , esRets, esWebSource)
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analysRets << analyRet
}
return analysRets;
}
def findMatchSameDatas(c, esWebSource){
def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true))
def siteDatas = thirdSeaweedMappingColl.find(query).toList()
def rows = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(siteDatas.size())
for(def i=0;i<siteDatas.size();i++){
final def d = siteDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def obj = findCommunities(d.url)
if(obj){
obj.match_id = d.match_id
rows << obj
}
// println d.city +" "+totalCount.get() +"/"+siteDatas.size()
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rows
}
def addFieldFromRonghe(c, totalCount, rh_analysRets){
@@ -209,14 +300,10 @@ class DistrictFieldAddBatch extends BaseBatch{
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
// continue
// }
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
@@ -224,7 +311,6 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
@@ -244,7 +330,9 @@ class DistrictFieldAddBatch extends BaseBatch{
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
if(newVal) {
newVal = newVal.toString()
}
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
@@ -312,12 +400,16 @@ class DistrictFieldAddBatch extends BaseBatch{
newValues.put(seaweedfield, newVal)
}
if(seaweedfield == "lng"){
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
try{
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
}
}catch(Exception e){
println esVal
}
}
}
@@ -361,6 +453,15 @@ class DistrictFieldAddBatch extends BaseBatch{
return list
}
static def findEsDatasByCityAndName(c, district, name, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' and region='${district}' and name='${name}'", {r->
list << r
})
return list
}
def fieldUpdate(total,updateRets){
def countMap = [:]
@@ -621,4 +722,26 @@ class DistrictFieldAddBatch extends BaseBatch{
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
}
}
def findCommunities(url){
if(!url){
return null
}
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
def rets = []
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
}catch(Exception e){
findCommunities(url)
}
def obj = null
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
obj = d
}
}
return obj;
}
}
@@ -0,0 +1,54 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class DistrictStatRentCleanBatch extends BaseBatch{
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
static main(args) {
runBatch(new DistrictStatRentCleanBatch(), args)
}
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def keys = CommonDatautils.getSimpleDistricts("district", c)
running(keys, c)
println c +" ÔËÐÐÍê³É"
}
}
def running(keys, c){
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
def fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("district_name", 1)
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
def delRets = []
for(def d : dbcursor){
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
println d.city+"@"+d.region+"@"+d.district_name
delRets << d._id
}
if(delRets.size()>=100){
def delquery = new BasicDBObject()
delquery.put("_id", new BasicDBObject('$in', delRets))
RENT_DIST_STAT_COLL.remove(delquery)
delRets.clear()
}
}
}
}
@@ -398,4 +398,32 @@ class FangjiaClientUtils {
return jsonObj;
}
/**
* 经纬度转换
* 1、高德转百度 2、高德转WGS84 3、百度转高德 4、WGS84转高德 5、百度转WGS84 6、WGS84转百度
*/
public static def convert(String locationstr, type){
def jsonObj = null
def content = null
def points = []
try {
def url = "http://192.168.0.19:31080/FangJiaNlpService/convert?locations="+URLEncoder.encode(locationstr,"UTF-8")+"&coordsys=${type}"
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
if(jsonObj && jsonObj.locations){
def locations = jsonObj.locations
for(def s : locations){
def arrs = s.split(",")
points = [arrs[0].toString().toDouble(), arrs[1].toString().toDouble()]
}
}
return points
} catch (Exception e) {
println e.getMessage()
}
return null;
}
}
@@ -0,0 +1,414 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 根据链家数据修改小区库中字段
* 使用到中间映射表
*/
@BatchOwner(Watcher.GaoJuan)
class UpdateDistrictField extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
def sources = ["链家", "我爱我家", "安居客"]
def statMap = [:]
static main(args) {
runBatch(new UpdateDistrictField(), args)
}
@Override
void process(Object args) {
def sourceMap = [:]
sources.each {source->
def valMap = findThirdMatchDist(source)
println source +" "+valMap.size()
if(valMap){
sourceMap.put(source, valMap)
}
}
def margeResults = megerDatas(sourceMap)
updateDB(margeResults)
println "修改完成 "+new Date().toString()
}
def megerDatas(sourceMap){
def set = new HashSet()
sources.each {source->
def valMap = sourceMap.get(source)
set.addAll(valMap.keySet())
}
def lastValues = []
set.each {id->
def dbObj = CommonDatautils.findDistrictById(id)
def datas = []
sources.each {source->
def valMap = sourceMap.get(source)
def sourceEsObj = findCommunities(valMap.get(id))
if(sourceEsObj){
if(source == "链家"){
sourceEsObj.source = "lianjia"
}else if(source == "我爱我家"){
sourceEsObj.source = "wiwj"
}else if(source == "安居客"){
sourceEsObj.source = "anjuke"
}
datas << sourceEsObj
}
}
if(dbObj && datas){
def newValMap = getNewField2(datas, dbObj)
if(newValMap){
newValMap.put("id", id)
lastValues << newValMap
}
}
}
statMap.each {source,fieldMap->
println source +" "+fieldMap.size()
fieldMap.each{f,v->
println f+" "+ v
}
println "----------------------------------------"
}
return lastValues
}
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
// def lianjisIds = lianjiaMap.keySet()
// def wiwjIds = wiwjMap.keySet()
// def anjukeIds = anjukeMap.keySet()
// def set = new HashSet()
// set.addAll(lianjisIds)
// set.addAll(wiwjIds)
// set.addAll(anjukeIds)
//
// def lastValues = []
// def lianjiaUpdateCount = 0
// def wiwjUpdateCount = 0
// def anjukeUpdateCount = 0
// def lianjiaCountMap = [:]
// def wiwjCountMap = [:]
// def anjukeCountMap = [:]
// set.each {id->
// BasicDBObject newValMap = new BasicDBObject()
// fields.each {field->
// def lianjiaValMap = lianjiaMap.get(id)
// def wiwjValMap = wiwjMap.get(id)
// def anjukeValMap = anjukeMap.get(id)
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
// newValMap.put(field, lianjiaValMap.get(field))
// lianjiaUpdateCount++
// def count = lianjiaCountMap.get(field)
// if(!count){
// lianjiaCountMap.put(field, 1)
// }else{
// lianjiaCountMap.put(field, count + 1)
// }
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
// newValMap.put(field, wiwjValMap.get(field))
// wiwjUpdateCount++
// def count = wiwjCountMap.get(field)
// if(!count){
// wiwjCountMap.put(field, 1)
// }else{
// wiwjCountMap.put(field, count + 1)
// }
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
// newValMap.put(field, anjukeValMap.get(field))
// anjukeUpdateCount++
// def count = anjukeCountMap.get(field)
// if(!count){
// anjukeCountMap.put(field, 1)
// }else{
// anjukeCountMap.put(field, count + 1)
// }
// }
// }
// newValMap.put("id", id)
// lastValues << newValMap
// }
//
// lianjiaCountMap.each {k,v->
// println "[链家] "+k+" "+v
// }
//
// wiwjCountMap.each {k,v->
// println "[我爱我家] "+k+" "+v
// }
//
// anjukeCountMap.each {k,v->
// println "[安居客] "+k+" "+v
// }
// return lastValues
// }
def updateDB(margeResults){
for(def mr : margeResults){
def id = mr.get("id")
mr.remove("id")
mr.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
}
}
// def batchUpdateDist(source){
//// def rets = findThirdMatchDist(source)
// println "[${source}] 符保条件小区量:"+rets.size()
// def updateIndex = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(rets.size())
// def allUpdateList = new CopyOnWriteArrayList()
//
// def sourceMap = new ConcurrentHashMap()
//
// for(def i=0;i<rets.size();i++) {
// final def ret = rets.get(i)
//// fixedThreadPool.execute(
//// new Runnable() {
//// void run() {
// try {
// def newValMap = running(ret.url, ret.old_match_id)
// if(newValMap){
// sourceMap.put(ret.old_match_id, newValMap)
// allUpdateList << newValMap
// updateIndex.addAndGet(1)
// }
// }catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
//// }
//// }
//// )
// }
// latch.await()
// println "总修改量:"+updateIndex.get()
// def countMap = [:]
// for(def map : allUpdateList){
// map.each{k,v->
// def count = countMap.get(k)
// if(!count){
// countMap.put(k, 1)
// }else{
// countMap.put(k, count+1)
// }
// }
// }
//
// countMap.each {k,v->
// println k +" 更新量:"+v
// }
//
// println "---------------------------------------------------------"
// return sourceMap
// }
def running(url, old_match_id){
def esObj = findCommunities(url)
def dbObj = CommonDatautils.findDistrictById(old_match_id)
if(dbObj){
def newValMap = getNewField(esObj, dbObj)
return newValMap
}
return null
}
def getNewField(esObj, dbObj){
BasicDBObject newValMap = new BasicDBObject()
esObj.each{field, esVal->
def dbVal = dbObj.get(field)
def newVal = null
if(esVal && field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(esVal && field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(esVal && field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
if(newVal){
newValMap.put(field, newVal)
}
}
return newValMap
}
def getNewField2(esDatas, dbObj){
BasicDBObject newValMap = new BasicDBObject()
def lastValueMap = [:]
def haveValueMap = [:]
fields.each{field->
for(def esData : esDatas){
def esVal = esData.get(field)
def dbVal = dbObj.get(field)
def newVal = null
if(haveValueMap.containsKey(field)){
continue
}else{
if(lastValueMap.containsKey(field)){
continue
}
if(esVal){
if(field=="estate_charge" || field=="car_site_count") {
esVal = FiledStrand.getNumber(esVal)
}
if(dbVal && esVal.toString()==dbVal.toString()){
haveValueMap.put(field, esVal)
}else{
if(field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
}
}
if(newVal){
lastValueMap.put(field, newVal)
//以下为统计各网站各字段的更新情况
def fieldStatMap = statMap.get(esData.source)
if(!fieldStatMap){
fieldStatMap = [:]
statMap.put(esData.source, fieldStatMap)
}
def count = fieldStatMap.get(field)
if(!count){
fieldStatMap.put(field, 1)
}else{
fieldStatMap.put(field, count + 1)
}
}
}
}
}
return lastValueMap
}
def findThirdMatchDist(source){
def query = new BasicDBObject("source",source).append("old_match_way", 1)
def keys = new BasicDBObject()
keys.put("url", 1)
keys.put("old_match_id", 1)
keys.put("old_match_city", 1)
keys.put("old_match_district", 1)
keys.put("old_match_name", 1)
def idsMap = [:]
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
for(def db : dbcursor){
if(!db.old_match_id && db.old_match_name){
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
if(dbo){
idsMap.put(dbo._id.toString(), db.url)
}
}else{
idsMap.put(db.old_match_id, db.url)
}
}
return idsMap
}
static def findCommunities(url){
if(!url){
return null
}
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
def rets = []
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
}catch(Exception e){
findCommunities(url)
}
def data = [:]
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
data.put("household_count", d.households)
data.put("loudong_count", d.loudong_cnt)
data.put("developer", d.developer)
data.put("estate_company", d.property_company)
data.put("estate_charge", d.property_fee)
data.put("car_site_count", d.parking_cnt)
}
}
return data;
}
}
@@ -26,7 +26,8 @@ class EsMonthHouseStat {
// task.statHouseUpdateCount()
// task.statDistHouse()
task.statRent()
// task.statRent()
task.statOfficeHouse()
}
def statRent(){
@@ -201,6 +202,51 @@ class EsMonthHouseStat {
},0)
}
def statOfficeHouse(){
def citys = CommonDatautils.findAllCity()
def allLists = []
allLists << ["城市", "近一年写字楼出售量", "写字楼量", "近一年出售房源量大于3的写字楼量", "大于3写字楼量占比"]
citys.each{c->
def count = findHouseCountByCat(c, "office_sell",20240101, 20250201)
def districts = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
def distCount = 0
if(count){
distCount = findHouseCountByCat(dist.city, dist.region, alias, 20240101, 20250201, "office_sell")
if(distCount && distCount>=3){
disthouseCount++
}
}
// println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
// println districts.size()
def precent = 0
if(districts){
precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
}
def row = [c , count, districts.size(), disthouseCount, precent+"%"]
println row
allLists << row
}
ExcelUtils.write("D:\\数据\\写字楼\\全国写字楼情况占比统计.xlsx", allLists, "写字楼情况")
}
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
def aliasStr = null;
@@ -219,6 +265,23 @@ class EsMonthHouseStat {
return total
}
def findHouseCountByCat(def city, def region, def alias, def startDate, def endDate, cat){
def aliasStr = null;
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') "
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
@@ -240,6 +303,17 @@ class EsMonthHouseStat {
}
def findHouseCountByCat(def city, def cat, def startDate, def endDate){
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}
@@ -32,8 +32,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
"淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12","房多多":"13","我爱我家":"14",
"禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20",
"乐有家":"21","幸福里":"22","中国房价行情":"23","常州住房":"24"
]
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
@@ -269,6 +272,33 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
if(third.get("csjj_esf_price")){
price_map.csjj_avg_price = third.get("csjj_esf_price")
}
if(third.get("leyoujia_esf_price")){
price_map.leyoujia_avg_price = third.get("leyoujia_esf_price")
}
if(third.get("c21_esf_price")){
price_map.c21_avg_price = third.get("c21_esf_price")
}
if(third.get("jiwu_esf_price")){
price_map.jiwu_avg_price = third.get("jiwu_esf_price")
}
if(third.get("xingfuli_esf_price")){
price_map.xingfuli_avg_price = third.get("xingfuli_esf_price")
}
if(third.get("zwfc_esf_price")){
price_map.zwfc_avg_price = third.get("zwfc_esf_price")
}
if(third.get("creprice_esf_price")){
price_map.creprice_avg_price = third.get("creprice_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -317,6 +347,20 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
web = "ÎÒ°®ÎÒ¼Ò"
}else if(k=="fccs_avg_price"){
web = "·¿²ú³¬ÊÐ"
}else if(k=="csjj_avg_price"){
web = "城市居家"
}else if(k=="leyoujia_avg_price"){
web = "乐有家"
}else if(k=="c21_avg_price"){
web = "C21"
}else if(k=="jiwu_avg_price"){
web = "吉屋网"
}else if(k=="xingfuli_avg_price"){
web = "幸福里"
}else if(k=="creprice_avg_price"){
web = "中国房价行情"
}else if(k=="zwfc_avg_price"){
web = "常州住房"
}
ret << sourceMap.get(web)
@@ -470,6 +514,33 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
}
if(!dataMap.get("leyoujia_esf_price") && db.get("leyoujia_esf_price") && db.get("leyoujia_esf_price")>3000){
dataMap.put("leyoujia_esf_price", db.get("leyoujia_esf_price"))
}
if(!dataMap.get("c21_esf_price") && db.get("c21_esf_price") && db.get("c21_esf_price")>3000){
dataMap.put("c21_esf_price", db.get("c21_esf_price"))
}
if(!dataMap.get("jiwu_esf_price") && db.get("jiwu_esf_price") && db.get("jiwu_esf_price")>3000){
dataMap.put("jiwu_esf_price", db.get("jiwu_esf_price"))
}
if(!dataMap.get("xingfuli_esf_price") && db.get("xingfuli_esf_price") && db.get("xingfuli_esf_price")>3000){
dataMap.put("xingfuli_esf_price", db.get("xingfuli_esf_price"))
}
if(!dataMap.get("creprice_esf_price") && db.get("creprice_esf_price") && db.get("creprice_esf_price")>3000){
dataMap.put("creprice_esf_price", db.get("creprice_esf_price"))
}
if(!dataMap.get("zwfc_esf_price") && db.get("zwfc_esf_price") && db.get("zwfc_esf_price")>3000){
dataMap.put("zwfc_esf_price", db.get("zwfc_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -38,8 +38,8 @@ class AddDistImagesBatch extends BaseBatch{
@Override
public void process(Object args) {
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def citys = CommonUtils.findAllCity()
citys.each {city->
def communitys = findAllEsData2(city, mdate)
@@ -47,14 +47,13 @@ class AddDistImagesBatch extends BaseBatch{
//第三方匹配小区
matchCommunitys(communitys)
}
statImgCount()
}
def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
//安居客小程序
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "贝壳找房"))
.must(date_range).must(date_range)
def rets = []
@@ -106,41 +105,69 @@ class AddDistImagesBatch extends BaseBatch{
def total = communitys.size()
for(def i=0;i<communitys.size();i++){
final def d = communitys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","贝壳找房").append("match_id", new BasicDBObject('$exists', true))
def obj = thirdSeaweedMappingColl.findOne(query)
if(obj) {
def vo = new BasicDBObject()
vo.put("city", obj.match_city)
vo.put("district", obj.match_district)
vo.put("name", obj.match_name)
if(d.out_images){
vo.put("out_pics", d.out_images)
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
if(obj.old_match_id && obj.match_id==obj.old_match_id){
def vo = new BasicDBObject()
vo.put("city", obj.match_city)
vo.put("district", obj.match_district)
vo.put("name", obj.match_name)
if(d.out_images && (!imageObj || (imageObj && !imageObj.out_pics))){
vo.put("out_pics", d.out_images)
}
if(d.house_type_images && (!imageObj || (imageObj && !imageObj.floorplan_pics))){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
if(vo.out_pics || vo.floorplan_pics){
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}
}
if(d.house_type_images){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}else{
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
});
}
latch.await()
return rets
}
def statImgCount(){
def q = new BasicDBObject()
def dbcursor = seaweed_fs_coll.find(q)
def out_pics_count = 0
def floor_count = 0
for(def d : dbcursor){
if(d.out_pics){
out_pics_count = out_pics_count + d.out_pics.size()
}
if(d.floorplan_pics){
d.floorplan_pics.each{k,v->
floor_count = floor_count + v.size()
}
}
}
println out_pics_count
println floor_count
}
}
@@ -13,18 +13,18 @@ class MsbPocTest {
task.run();
}
def run(){
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
def inputfile = "D:\\数据\\建行\\房产估值\\房讯通10月价格核查_1126-FXT提交结果20241205.xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
// matchOld(inputfile)
matchOld(inputfile)
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
// test("D:\\数据\\美术宝\\test2.xlsx")
// test("D:\\数据\\美术宝\\FiveLevelTest.xlsx")
// cityPrice()
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
addCCBPrice(inputfile)
// addCCBPrice(inputfile)
}
def cityPrice(){
@@ -69,24 +69,24 @@ class MsbPocTest {
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Y'] && row['G']){
if(!row['AD'] ){
def city = row['C']
def region = null //row['D']
def name = row['E']
def address = row['G']
def name = null// row['E']
def address = row['D']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("propertyType")
row['AD'] = jsonObject.get("id")
row['AE'] = jsonObject.get("city")
row['AF'] = jsonObject.get("district")
row['AG'] = jsonObject.get("name")
row['AH'] = jsonObject.get("propertyType")
// row['X'] = jsonObject.get("credit")
}
}
})
},1)
}
@@ -114,7 +114,7 @@ class MsbPocTest {
if(!row['Y']){
def city = row['C']
def region = null //row['B']
def address = row['G'] //row['G']//row['F']
def address = row['D'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
@@ -125,7 +125,7 @@ class MsbPocTest {
row['AB'] = jsonObject.get("name")
}
}
})
},1)
}
@@ -38,9 +38,9 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
def filePath = "D:\\Êý¾Ý\\¼Û¸ñ\\»ÝÖÝ.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
@@ -174,26 +174,28 @@ class AddPriceTest {
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['U']){
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(d['A']){
def cityId = cityCodeDataMap.get(d['B'])
def map = findAvgPrice(d['A'], cityId)
if(map){
d['H'] = map.avg_price
d['I'] = map.calc_type
d['I'] = map.avg_price
d['J'] = map.month
// d['H'] = map.calc_type
// d['AM'] = map.credit_value
}
// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId)
// if(newPriceMap){
// d['Q'] = newPriceMap.avg_price
// }
def newPriceMap = findNewFjwAvgPrice(d['A'], cityId)
if(newPriceMap){
d['G'] = newPriceMap.avg_price
d['H'] = newPriceMap.month
}
}
})
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -213,7 +215,7 @@ class AddPriceTest {
def findNewFjwAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance235.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -270,14 +272,14 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['B']) {
if (!d['H']) {
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
if(gxdPrice){
d['B'] = gxdPrice.price
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
d['H'] = gxdPrice.price
d['I'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
}
}
@@ -305,11 +307,11 @@ class AddPriceTest {
ExcelUtils.multiThreadingModify(path,{d->
// if(!d['L']){
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['AJ'])
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
d['E'] = dist.name
// d['E'] = dist.name
// d['D'] = dist.status
// d['E'] = dist.name_cat
// d['F'] = dist.estate_type2
@@ -318,9 +320,9 @@ class AddPriceTest {
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['G'] = dbo.avg_price
d['E'] = dbo.avg_price_2b
// d['X'] = dbo.cal_type
// d['E'] = dbo.avg_price_2b
// d['E'] = dbo.avg_price
}
}
// }
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -17,7 +18,7 @@ import com.mongodb.DBCollection
*/
@BatchOwner(Watcher.GaoJuan)
//@BatchOwner(Watcher.GaoJuan)
class RelatedTaiWanBatch extends BaseBatch{
@@ -69,7 +70,9 @@ class RelatedTaiWanBatch extends BaseBatch{
if(r.lon && r.lat){
for(def m : zimaoqu_range_datas){
// def flag = InsidePolygon.isInArea(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
List<List<Point>> pointsList = InsidePolygon.createPoint4(m.shape)
// def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, pointsList)
def flag = InsidePolygon.isInArea(r.lon, r.lat, pointsList)
if(flag){
resKey = m.name
break
@@ -226,16 +226,17 @@ class SeaweedDataSynch extends BaseBatch{
for(def key : dist.mapping_alias){
if(key && !key.toString().equals("null")){
aliasNames << key
if(aliasNames.size()>5){
map.alias = aliasNames
break;
}else{
if(aliasNames){
map.alias = aliasNames
}
}
// if(aliasNames.size()>5){
// map.alias = aliasNames
// break;
// }else{
// if(aliasNames){
// map.alias = aliasNames
// }
// }
}
}
map.alias = aliasNames
}
if(dist.addr_alias){
@@ -243,16 +244,17 @@ class SeaweedDataSynch extends BaseBatch{
for(def key : dist.addr_alias){
if(key && !key.toString().equals("null")){
names << key
if(names.size()>5){
map.addr_alias = names
break;
}else{
if(names){
map.addr_alias = names
}
}
// if(names.size()>5){
// map.addr_alias = names
// break;
// }else{
// if(names){
// map.addr_alias = names
// }
// }
}
}
map.addr_alias = names
}
map.category = dist.cat
@@ -0,0 +1,132 @@
package com.fangjia.script.looplineinfo
import com.fangjia.script.common.DataHandleUtils
import java.awt.geom.Point2D;
class InPolygon {
public static List<List<Point2D.Double>> createPoint(def pointList){
List<List<Point2D.Double>> alllist = new ArrayList<>();
for(def pointArr : pointList){
List<Point2D.Double> list = new ArrayList<>();
for(def arr : pointArr){
double x = DataHandleUtils.getDoubleRound(6, arr[0]);
double y = DataHandleUtils.getDoubleRound(6, arr[1]);
Point2D.Double point = new Point2D.Double(x,y);
list.add(point);
}
alllist << list
}
return alllist
}
public static def isInArea(Point2D.Double point, List<List<Point2D.Double>> pointList){
def flag = false;
for(List<Point2D.Double> pointArr : pointList){
flag = isPtInPoly(point, pointArr)
if(flag){
break;
}
}
return flag;
}
/**
* 判断点是否在多边形内
* <p>
* 整个算法的思路为:作点平行于y轴的射线,这样就可以直接比较交点是否大于点(x,y)中y的值,可以简化判断。
*
* @param point 检测点
* @param pts 多边形的顶点
* @return 点在多边形内返回true, 否则返回false
*/
public static boolean isPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
int polygonSidesCount = pts.size();
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
boolean boundOrVertex = true;
// 射线与多边形相交的次数
int intersectCount = 0;
//浮点类型计算时候与0比较时候的容差
double precision = 2e-10;
// 相邻两点
Point2D.Double p1, p2;
// 当前点
Point2D.Double p = point;
// 线段起点
p1 = pts.get(0);
for (int i = 1; i <= polygonSidesCount; ++i) {
//按照点录入的顺序依次检查相邻两点组成的线段和当前的点的关系。
if (p.equals(p1)) {
//如果当前点就是多边形的顶点之一,直接返回。
return boundOrVertex;
}
// 线段终点
p2 = pts.get(i % polygonSidesCount);
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
//点在x轴上的映射,明显超出了线段在x轴上的投影。此时冲着上做射线,肯定没有交点,直接跳过。
p1 = p2;
continue;
}
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
// 当前点在线段于x轴上的投影内
if (p.y <= Math.max(p1.y, p2.y)) {
//当前点的 y 坐标小于 线段对y轴投影的最大值
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
// 若线段同样是平行于y轴,则可以断定,当前点,在多边形的这条垂直于x轴的边上。
return boundOrVertex;
}
if (p1.y == p2.y) {
// 若线段为平行于x轴的水平线
if (p1.y == p.y) {
//当前点正好位于该水平线上,直接返回该点位于多边形的一条边上。
return boundOrVertex;
} else {
//如果当前点在水平线以下,增加一个交点。
++intersectCount;
}
} else {
// 如果不是水平线,则用两点式求当前点的x带入多边形线段的直线方程后,对应的y的坐标
double xInSideLineFormulaResultY = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
if (Math.abs(p.y - xInSideLineFormulaResultY) < precision) {
// 误差允许范围内,改点就在线段上,则表明,该点位于多边形的一个边上。
return boundOrVertex;
}
if (p.y < xInSideLineFormulaResultY) {
// 如果线段上取得的y比当前点的y要大,当前做向上的射线,肯定交于上方的一个点。
++intersectCount;
}
}
}
} else {
// 当前点不在线段投影到x轴的区间中
if (p.x == p2.x && p.y <= p2.y) {
// 但恰好位于线段终点的x坐标对应的平行于y轴上的线上的低于终点y的一点
// 此时检查下一点能否将其x边界包含。
Point2D.Double p3 = pts.get((i + 1) % polygonSidesCount);
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
// 若当前点的x坐标位于 p1和p3组成的线段关于x轴的投影中,则记为该点的射线只穿过端点一次。
++intersectCount;
} else {
// 若当前点的x坐标不能包含在p1和p3组成的线段关于x轴的投影中,则点射线通过的两条线段组成了一个弯折的部分,
// 此时我们记射线穿过该端点两次
intersectCount += 2;
}
// 此判断的核心思路是由点在两条线段的内部还是外部去思考得出的
}
}
// 进行下一个线段的判断
p1 = p2;
}
if (intersectCount % 2 == 0) {
//偶数在多边形外
return false;
} else {
//奇数在多边形内
return true;
}
}
}
@@ -324,4 +324,8 @@ class InsidePolygon {
}
return name;
}
}
@@ -1,72 +0,0 @@
package com.fangjia.script.majintao
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
String excelFilePath = "F:\\估家\\文件\\北交所\\testcopy1.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
String comprehensiveUrl = "http://localhost:9180/comprehensiveQuery/houseValueInformation";
String communityUrl = "http://open.fangjia.com/addrSvc/avgPrice";
String valueUrl = "http://open.fangjia.com/property/addressEvaluate";
String caseUrl = "http://open.fangjia.com/house/case";
String poiUrl = "http://open.fangjia.com/district/addrFacilitys";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
String city = items['A']
String address = items['B']
double size = items['C']
System.out.println(city + "---" + address + "----" + size);
map.put("city", city)
map.put("address", address)
map.put("size", size.toString())
map.put("strctureArea", size.toString())
map.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB")
if (!address.contains("%")) {
// 发送HTTP请求
long start
long end
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(communityUrl, map, "UTF-8")
//
//
// end = System.currentTimeMillis();
// items['D'] = String.valueOf(end - start);
//
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(valueUrl, map, "UTF-8")
// end = System.currentTimeMillis();
// items['E'] = String.valueOf(end - start);
//
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(caseUrl, map, "UTF-8")
// end = System.currentTimeMillis();
// items['F'] = String.valueOf(end - start);
//
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(poiUrl, map, "UTF-8")
// end = System.currentTimeMillis();
// items['G'] = String.valueOf(end - start);
start = System.currentTimeMillis();
def content = httpRequest.getJsonContent(comprehensiveUrl, map, "UTF-8")
end = System.currentTimeMillis();
items['H'] = String.valueOf(end - start);
println content.get("msg")
if(content.get("msg").contains("该地址不支持估价!")) {
items['I'] = "该地址不支持估价!"
}
}
})
@@ -1,4 +0,0 @@
import org.bson.types.ObjectId
a = new ObjectId()
print(a)
@@ -1,28 +1,16 @@
package com.fangjia.script.trade
import java.util.regex.Matcher
package com.fangjia.script.sync
import com.fangjia.core.Constants
import com.fangjia.script.BaseBatch;
import com.fangjia.script.BatchOwner
import com.fangjia.script.util.DateUtils;
import java.util.regex.Pattern
import org.apache.commons.lang.StringUtils;
import com.fangjia.core.util.StringUtil;
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBObject
import com.fangjia.script.util.MongoUtils;
import com.fangjia.core.util.MongoUtils;
import com.fangjia.script.Watcher;
import com.mongodb.Bytes
import org.apache.commons.io.FileUtils
/**
* ½«ÏßÉÏopen uriÊý¾Ýͬ²½µ½192.168.0.60ÉÏ
* @author gaojuan
@@ -31,7 +19,7 @@ import org.apache.commons.io.FileUtils
class CopyOpenUriLog extends BaseBatch{
static DBCollection DESC_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com", 27020)
static DBCollection DESC_OPEN = MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
public static final DBCollection SOURCE_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "log.mongo.db.fangjia.com", 52002)
static main(String[] args){
runBatch(new CopyOpenUriLog(), args)
@@ -1062,6 +1062,7 @@ class DataStandardBatch extends BaseBatch{
q.put("s_date", SHORT_DATE);
DBObject udata2 = new BasicDBObject()
udata2.put("match_id", district._id.toString());
udata2.put("match_city", district.city);
udata2.put("match_region", district.region);
udata2.put("match_name", district.name);
@@ -27,7 +27,9 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["搜房":"搜房"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
def esSources = ["安居客小程序":"安居客","搜房":"搜房","诸葛找房APP":"诸葛找房","幸福里APP":"幸福里","吉屋":"吉屋","中国房价行情":"中国房价行情","贝壳找房":"贝壳找房",
"链家":"链家","C21":"C21","城市房产小程序":"城市房产","乐有家":"乐有家","我爱我家":"我爱我家","中原地产":"中原地产","兔博士PC端":"兔博士",
"365淘房":"365淘房","Q房":"Q房","麦田":"麦田","常州住房":"常州住房"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
@@ -33,7 +33,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf", "房天下":"fang_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -9,13 +9,8 @@ import net.sf.json.JSONObject
class BaiduMapUtils {
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
def static AKS = ["hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
"B42b12d01f2dee712fd66c3e886158a4", "1a3cde429f38434f1811a75e1a90310c","W3M5uHfcrmlAdhQjf6v93LCI",
"C8bb035aacf602551e829e4c64575224", "B42b12d01f2dee712fd66c3e886158a4",
"1a3cde429f38434f1811a75e1a90310c","9E99D3540e4dafb7894f7b9fe27fa6b5"];
private static String AK = "1a3cde429f38434f1811a75e1a90310c";
def static AKS = ["1a3cde429f38434f1811a75e1a90310c","1a3cde429f38434f1811a75e1a90310c"];
public static boolean HAS_AVAILABLE_AK = true;
static def getAddress(lng, lat) throws Exception{
@@ -83,8 +78,8 @@ class BaiduMapUtils {
throw new RuntimeException("无可用百度AK");
}
System.out.println("更换AK");
for(int i = 0 ;i < AKS.length; i++){
if(i == AKS.length - 1){
for(int i = 0 ;i < AKS.size(); i++){
if(i == AKS.size() - 1){
AK = AKS[0];
HAS_AVAILABLE_AK = false;
throw new RuntimeException("无可用百度AK");
@@ -127,6 +122,35 @@ class BaiduMapUtils {
return null
}
public static JSONObject geocoding(locationstr) {
String url = "https://api.map.baidu.com/reverse_geocoding/v3/?ak="+AK+"&output=json&coordtype=gcj02ll&location="+locationstr
boolean stop = false;
int recount = 0;
JSONObject jsonObj = null
while(!stop && recount<6) {
try{
HttpRequest request = new HttpRequest();
try{
String result = request.get(url,null,"UTF-8");
def json = JSONObject.fromObject(result);
if(json.getInt("status") >= 300){
changeAK();
recount++
}else{
jsonObj = json.result
}
}catch (Exception e){
e.printStackTrace();
}
}catch (Exception e){
recount++;
println e.getMessage()
}
}
return jsonObj
}
public static def searchByBounds(key, min_lat, min_lng, max_lat, max_lng){
boolean stop = false;
int recount = 0;
@@ -1,6 +1,7 @@
package com.fangjia.script.util
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@@ -192,6 +193,19 @@ public class CommonUtils {
}
}
static void recalcAverage(String city, String region, String name, Integer count =1 , String category="sell", Boolean retainHistory=true){
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate";
def params = new HashMap<String,String>();
params.city = city;
params.region = region;
params.name = name;
params.count = count.toString();
params.category = category;
params.retainHistory = retainHistory.toString();
HttpRequest request = new HttpRequest();
println(request.get(url, params, "UTF-8"))
}
/**
* 重算小区均价
* @param city
@@ -221,7 +235,7 @@ public class CommonUtils {
cmd.put("name", name)
cmd.put('estate_type', estateType)
cmd.put("calc_date", calcDate)
cmd.put("calc_type", 0)
cmd.put("calc_type", calcType)
println(cmd)
DBObject data = new BasicDBObject()
data.put("category", "sell")
@@ -1,4 +1,8 @@
package com.fangjia.script.util
import java.text.DecimalFormat
class GeoHelper {
@@ -60,10 +64,159 @@ class GeoHelper {
// return GetLatLon(a, distance, angle);
// }
//
public static void main(String[] args){
GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
}
public static haversine(lat1, lon1, lat2, lon2) {
def R = 6371; // 地球半径(公里)
def lat1Rad = lat1 * (Math.PI / 180);
def lat2Rad = lat2 * (Math.PI / 180);
def deltaLat = (lat2 - lat1) * (Math.PI / 180);
def deltaLon = (lon2 - lon1) * (Math.PI / 180);
def a = Math.sin(deltaLat / 2) ** 2 +
Math.cos(lat1Rad) * Math.cos(lat2Rad) *
Math.sin(deltaLon / 2) ** 2;
def c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
return R * c; // 返回距离(公里)
}
public static calculateArea(coords) {
def area = 0;
def n = coords.size();
for (def i = 0; i < n; i++) {
def j = (i + 1) % n;
area += coords[i][0] * coords[j][1];
area -= coords[j][0] * coords[i][1];
}
area = Math.abs(area) / 2; // 平方度
// 转换为平方公里
def conversionFactor = 111.32; // 大约每度经纬度的公里数(考虑到地球的曲率)
def df = new DecimalFormat("#.000000")
return Double.parseDouble(df.format(area * conversionFactor * conversionFactor)) // 返回面积(平方公里)
// return area * conversionFactor * conversionFactor
}
public static void main(String[] args){
// GeoHelper.GetLatLon( 31.222361, 121.380462, 0.6, 0);
// GetDistance(31.222361, 121.380462, 31.227757176233904, 121.380462);
List<List<Double>> coordinates = [
[
117.61878540582734,
24.667331307258223
],
[
117.61400840589488,
24.670008244935104
],
[
117.60936140565889,
24.671837202755682
],
[
117.60834440585768,
24.668667276729821
],
[
117.60696440583342,
24.667256309201207
],
[
117.60517040553236,
24.667762297657362
],
[
117.60272340547806,
24.660100475272998
],
[
117.60254340555321,
24.658630509428463
],
[
117.60206940590035,
24.654781598282568
],
[
117.60254840557604,
24.652731645907636
],
[
117.60346440599301,
24.651555673447035
],
[
117.6070314057805,
24.652578649754957
],
[
117.61605140571805,
24.651939664372833
],
[
117.6156454058372,
24.654165613124405
],
[
117.61559340595863,
24.655023593107103
],
[
117.61535440558418,
24.658990501065318
],
[
117.61662340528017,
24.659352492642654
],
[
117.61678740531192,
24.661365445721003
],
[
117.61472940542001,
24.661623439403797
],
[
117.61462940586114,
24.661837434759093
],
[
117.61470140583106,
24.662665415879701
],
[
117.61487240553515,
24.663165403873506
],
[
117.61522940591078,
24.663494396452791
],
[
117.6157014059138,
24.663523395394918
],
[
117.61627240565099,
24.663438397829282
],
[
117.61693540544979,
24.663181403867725
],
[
117.61878540582734,
24.667331307258223
]
]
double area = calculateArea(coordinates)
println "The area of the polygon is: ${String.format("%.2f", area)} square kilometers"
}
}
@@ -43,15 +43,15 @@ class Test {
static main(args) {
// String test1 = " qwdwqd ssss ";
// String test2 = " qwdwqd ss";
// String FiveLevelTest = " qwdwqd ss";
// String test3 = "qwdwqd dd";
//
// test1 = test1.replaceAll(" |\\s+", " ");
// test2 = test2.replaceAll(" |\\s+", " ");
// FiveLevelTest = FiveLevelTest.replaceAll(" |\\s+", " ");
// test3 = test3.replaceAll(" |\\s+", " ");
//
// String[] arrs1 = test1.split("\\s+")
// String[] arrs2 = test2.split("\\s+")
// String[] arrs2 = FiveLevelTest.split("\\s+")
// String[] arrs3 = test3.split("\\s+")
//
// println arrs1
@@ -0,0 +1,50 @@
package com.fangjia.script.wenyan
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.Document
import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.get.GetResponse
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
////def data = []
////data << ["ID","城市","区域","名称","地址", "财力评级"]
//RestHighLevelClient client = ElasticUtils.getClient89("192.168.0.89")
//district_analy_data_client = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data")
//level_map = ["1": "S", "2": "A", "3": "B", "4": "C", "5": "D"]
//level_maps = new HashMap<>()
//for(def key in level_map.keySet()){
// level_maps.put(level_map[key], key)
//}
//ExcelUtils.modify("C:\\Users\\goojia\\Desktop\\财力评级-厦门.xlsx",{List item->
//// GetResponse response = client.get(new GetRequest("community_summary", item["A"] as String), RequestOptions.DEFAULT)
//// def query = new Document("city", item['B'].toString().replace("市", ""))
//// query.append("region", item['C'].replace("区", ""))
//// query.append("district_name", item["D"])
//// def stat = district_analy_data_client.find(query).first()
//// println(query)
//// item["F"] = response.getSourceAsMap()?.consumption_rating
//// item["G"] = stat?.community_rating
// item["G"] = level_maps.get(item["G"], null)
//// println(item)
//})
//System.exit(0)
////for (def region in ["思明", "翔安", "湖里"]) {
//// DbUtils.seaweed.find(new BasicDBObject("city", "厦门").append("region", region).append("cat", "district").append("status", 0)).each { BasicDBObject dbo ->
//// println("${dbo._id.toString()}\t${dbo.city}市\t${dbo.region}区\t${dbo.name}\t${dbo.address}")
//// }
////}
ExcelUtils.read("C:\\Users\\goojia\\Desktop\\海南省城市均价.xlsx",1).each {List item->
def yearMonth = DateUtil.formatByPattern(item[0], "yyyyMM").toInteger()
println(item)
}
@@ -0,0 +1,15 @@
package com.fangjia.script.wenyan.xiamen
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.multiThreadingModify("D:\\本地化部署\\厦门银行数据\\网点经纬度业务选择网点.xlsx",{List item ->
def location = item['E']
if(location){
def lng = location.toString().split(",")[0].toDouble()
def lat = location.toString().split(",")[1].toDouble()
def bounds = GeoUtils.getAround(lat, lng, 3000)
item["G"] = "${DataUtil.getRound(6, bounds[2])},${DataUtil.getRound(6,bounds[0])},${DataUtil.getRound(6,bounds[3])},${DataUtil.getRound(6,bounds[1])}"
}
})
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //±¦½£
//94c74c60e102972a8d17b7b75d7403be
def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //±¦½£:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995£ºdfc2245bac8a7b2d5b62e060e59d4eca
/**
@@ -29,12 +30,28 @@ class GaoDeApiUtils {
params.put("location", location)
params.put("extensions","all")
String content
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
def stop = true
JSONObject jsonObj = null
def errorCount = 1
while (stop){
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
jsonObj = JSONObject.fromObject(content)
if(jsonObj.status!="1"){
println jsonObj
errorCount ++
Thread.sleep(3000)
if(errorCount>3){
stop = false
}
}else{
stop = false
}
}
JSONObject jsonObj = JSONObject.fromObject(content)
return jsonObj
}