Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -537,6 +537,24 @@
|
||||
<version>1.70</version> <!-- 确保使用最新版本 -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
<artifactId>jts-core</artifactId>
|
||||
<version>1.18.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-text</artifactId>
|
||||
<version>1.9</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.hankcs.hanlp.restful</groupId>
|
||||
<artifactId>hanlp-restful</artifactId>
|
||||
<version>0.0.12</version>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>com.fangjia</groupId>-->
|
||||
<!-- <artifactId>fangjia-open</artifactId>-->
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script
|
||||
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.*
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -112,6 +112,18 @@ abstract class BaseBatch {
|
||||
}
|
||||
if(SystemUtils.IS_OS_LINUX){
|
||||
sendDingDingMsg("2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c",msg)
|
||||
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("phone", "13764369683");
|
||||
params.put("type", "taskFail");
|
||||
params.put("msg", batchName.substring(batchName.length() - 20, batchName.length()));
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8");
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
|
||||
task.exception = msg.toString()
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ import java.sql.DriverManager
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo", 27017)
|
||||
static void main(String[] args) {
|
||||
runBatch(new ClickHouseDistrictStatMarkupCalcBatch(), args)
|
||||
}
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ class DatabaseStatWatcherBatch extends BaseBatch {
|
||||
def yesterday = DateUtil.add(today, -1)
|
||||
|
||||
for(def host : hosts){
|
||||
Mongo mongo = new Mongo(new MongoURI("mongodb://${host}"))
|
||||
Mongo mongo = new Mongo(new MongoURI("mongodb://developer:G00IDCDEV12385@${host}:28017"))
|
||||
for(def databaseName : mongo.getDatabaseNames()){
|
||||
if(NOT_WATCH_DBS.contains(databaseName)){
|
||||
continue
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import com.mongodb.DBCollection
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class DistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 27017)
|
||||
Date oneYearAgo = DateUtil.add(new Date(), Calendar.MONTH, -1)
|
||||
|
||||
|
||||
|
||||
+5
-4
@@ -1,16 +1,17 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
//没有成交时间,随机生成
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
*/
|
||||
class AddThirdPriceSample {
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
+2
-3
@@ -43,7 +43,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -71,8 +71,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
|
||||
+10
-10
@@ -61,25 +61,25 @@ class CcbCommonUtils {
|
||||
def countStr = args[0]
|
||||
if(countStr=="344"){
|
||||
citys = CcbCommonUtils.citys_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="92"){
|
||||
citys = CcbCommonUtils.citys_92
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="252"){
|
||||
citys = CcbCommonUtils.citys_252
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="all"){
|
||||
citys = CcbCommonUtils.citys_fjw_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}
|
||||
}else{
|
||||
citys = CcbCommonUtils.citys_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}
|
||||
println "运行城市数量:"+citys.size()
|
||||
return citys;
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ class DistrictDelete {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
|
||||
+6
-5
@@ -1353,13 +1353,14 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
//},0)
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\更新人工指导价_0314.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
d['C'] = dist.city
|
||||
d['D'] = dist.region
|
||||
d['E'] = dist.name
|
||||
d['F'] = dist.status
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
+66
-3
@@ -27,11 +27,14 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
|
||||
"淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12","房多多":"13","我爱我家":"14",
|
||||
"禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20",
|
||||
"乐有家":"21","幸福里":"22","中国房价行情":"23","常州住房":"24"
|
||||
]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -263,6 +266,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("csjj_esf_price")){
|
||||
price_map.csjj_avg_price = third.get("csjj_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("leyoujia_esf_price")){
|
||||
price_map.leyoujia_avg_price = third.get("leyoujia_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("c21_esf_price")){
|
||||
price_map.c21_avg_price = third.get("c21_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("jiwu_esf_price")){
|
||||
price_map.jiwu_avg_price = third.get("jiwu_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("xingfuli_esf_price")){
|
||||
price_map.xingfuli_avg_price = third.get("xingfuli_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("zwfc_esf_price")){
|
||||
price_map.zwfc_avg_price = third.get("zwfc_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("creprice_esf_price")){
|
||||
price_map.creprice_avg_price = third.get("creprice_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -313,6 +340,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房产超市"
|
||||
}else if(k=="csjj_avg_price"){
|
||||
web = "城市居家"
|
||||
}else if(k=="leyoujia_avg_price"){
|
||||
web = "乐有家"
|
||||
}else if(k=="c21_avg_price"){
|
||||
web = "C21"
|
||||
}else if(k=="jiwu_avg_price"){
|
||||
web = "吉屋网"
|
||||
}else if(k=="xingfuli_avg_price"){
|
||||
web = "幸福里"
|
||||
}else if(k=="creprice_avg_price"){
|
||||
web = "中国房价行情"
|
||||
}else if(k=="zwfc_avg_price"){
|
||||
web = "常州住房"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -603,6 +642,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
|
||||
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("leyoujia_esf_price") && db.get("leyoujia_esf_price") && db.get("leyoujia_esf_price")>3000){
|
||||
dataMap.put("leyoujia_esf_price", db.get("leyoujia_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("c21_esf_price") && db.get("c21_esf_price") && db.get("c21_esf_price")>3000){
|
||||
dataMap.put("c21_esf_price", db.get("c21_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("jiwu_esf_price") && db.get("jiwu_esf_price") && db.get("jiwu_esf_price")>3000){
|
||||
dataMap.put("jiwu_esf_price", db.get("jiwu_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("xingfuli_esf_price") && db.get("xingfuli_esf_price") && db.get("xingfuli_esf_price")>3000){
|
||||
dataMap.put("xingfuli_esf_price", db.get("xingfuli_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("creprice_esf_price") && db.get("creprice_esf_price") && db.get("creprice_esf_price")>3000){
|
||||
dataMap.put("creprice_esf_price", db.get("creprice_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("zwfc_esf_price") && db.get("zwfc_esf_price") && db.get("zwfc_esf_price")>3000){
|
||||
dataMap.put("zwfc_esf_price", db.get("zwfc_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ class TradeStatLog {
|
||||
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
|
||||
+4
-4
@@ -18,10 +18,10 @@ import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
|
||||
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "mongo.db.fangjia.com")
|
||||
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ class MatchDealPriceRegion {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
|
||||
*/
|
||||
class EvaluationBatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ class FumingTest {
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
|
||||
+1
-1
@@ -38,7 +38,7 @@ class HailanTest {
|
||||
}
|
||||
|
||||
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class PocStat {
|
||||
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
|
||||
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ class RuiFenPocTest {
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
|
||||
+142
-13
@@ -7,6 +7,7 @@ import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.newDistrict.InvokeHttpRequest
|
||||
import com.fangjia.script.util.*
|
||||
@@ -98,11 +99,13 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com", 28017)
|
||||
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -259,7 +262,7 @@ class CommonDatautils {
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","托斯卡纳庄园")
|
||||
// q.put("name","新泾三村")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -597,9 +600,9 @@ class CommonDatautils {
|
||||
keys.put("region", 1);
|
||||
keys.put("name", 1);
|
||||
keys.put("_id", 1);
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
|
||||
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
|
||||
Iterator<DBObject> iters = cursor.iterator()
|
||||
def result = [];
|
||||
def result = new HashSet();
|
||||
try{
|
||||
while(iters.hasNext()) {
|
||||
DBObject obj = iters.next()
|
||||
@@ -755,7 +758,10 @@ class CommonDatautils {
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("region", region);
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
|
||||
q.put("name", name);
|
||||
q.put("cat", cat)
|
||||
q.put("status", 0)
|
||||
@@ -786,7 +792,8 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
@@ -814,7 +821,9 @@ class CommonDatautils {
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
if(!d.township){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
@@ -1131,6 +1140,26 @@ class CommonDatautils {
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static def findNormalDistrict(String city, String region, String name, String cat, List fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("cat", cat);
|
||||
q.put("name", name);
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
q.put("status",0)
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
return SEAWEED_COLL.findOne(q, keys);
|
||||
}
|
||||
|
||||
public static def findDistrictsByCityAndName(String city, String name, String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
@@ -4209,19 +4238,21 @@ class CommonDatautils {
|
||||
|
||||
|
||||
static def findAllNewPoiDatas(city){
|
||||
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
|
||||
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
index++
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
|
||||
// }
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def findOneThirdNetWorkOffice(city, region, name){
|
||||
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
|
||||
def map = [:]
|
||||
@@ -4234,6 +4265,16 @@ class CommonDatautils {
|
||||
return map
|
||||
}
|
||||
|
||||
static def findThirdNetWorkOffices(city, region, name){
|
||||
String sql = "select * from office where city='${city}' and region='${region}' and name='${name}'"
|
||||
def datas = []
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql, { m->
|
||||
datas << m
|
||||
})
|
||||
return datas
|
||||
}
|
||||
|
||||
|
||||
static def findThirdNetWorkOffice(){
|
||||
String sql = "select city,region,name,address,location from office where city='烟台' or city='北京'"
|
||||
@@ -4678,6 +4719,14 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def httpSaveData89(def newUrl,def params){
|
||||
def header = [:]
|
||||
header.put("Authorization","Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString())
|
||||
doPost(newUrl,header, params, true, "utf-8")
|
||||
}
|
||||
|
||||
|
||||
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
try{
|
||||
@@ -4965,6 +5014,20 @@ class CommonDatautils {
|
||||
return regionMap
|
||||
}
|
||||
|
||||
static def queryRangeByCat(cat){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type",cat)
|
||||
q.put("shape",new BasicDBObject('$exists', true))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city_short",1)
|
||||
keys.put("district_short",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList()
|
||||
}
|
||||
|
||||
|
||||
static def queryAllRegionNameByCity(){
|
||||
|
||||
@@ -5135,21 +5198,66 @@ class CommonDatautils {
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def distName = null
|
||||
if(d.mzb_district_name){
|
||||
def list = townMap.get(d.mzb_district_name)
|
||||
distName = d.mzb_district_name
|
||||
}else{
|
||||
distName = d.district
|
||||
}
|
||||
if(distName){
|
||||
def list = townMap.get(distName)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(d.mzb_district_name, list)
|
||||
townMap.put(distName, list)
|
||||
}
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryAllTown(fields){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
for(def key : fields){
|
||||
keys.put(key,1)
|
||||
}
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def townMap = cityMap.get(d.city)
|
||||
if(!townMap){
|
||||
townMap = [:]
|
||||
cityMap.put(d.city, townMap)
|
||||
}
|
||||
|
||||
def distName = null
|
||||
if(d.mzb_district_name){
|
||||
distName = d.mzb_district_name
|
||||
}else{
|
||||
distName = d.district
|
||||
}
|
||||
if(distName){
|
||||
def list = townMap.get(distName)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(distName, list)
|
||||
}
|
||||
}
|
||||
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return cityMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryFullRegionTownByCity(city, fields){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
@@ -5515,4 +5623,25 @@ class CommonDatautils {
|
||||
return 0
|
||||
}
|
||||
|
||||
static def findBusinessByCity(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("short_city", city)
|
||||
return bk_business_coll.find(q).toList()
|
||||
}
|
||||
|
||||
|
||||
static def findOfficeByMap(city, small_type){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("small_type", small_type))
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "gaode_map_poi_2024", queryBuilder, { String id, Map map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
|
||||
index++
|
||||
rets << ["map_id":map.data_id,"province":map.province,"city":map.city, "district":map.district, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.lon, "lat":map.location.lat, "parent_id":map.parent]
|
||||
// }
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ import org.apache.commons.lang.StringUtils
|
||||
class FewerHouseDistrictCrawlCommandBatch extends BaseBatch {
|
||||
private static
|
||||
final DBCollection ARTEMIS_COLL = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com")
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new FewerHouseDistrictCrawlCommandBatch(), args)
|
||||
|
||||
@@ -15,7 +15,7 @@ import org.bson.types.ObjectId
|
||||
//new FewerHouseDistrictCrawlCommandBatch().process(null);
|
||||
//return;
|
||||
//
|
||||
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "192.168.0.43", 27017);
|
||||
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com", 27017);
|
||||
//DBCursor cursor_artemis_message = col_43_artemis_message.find(new BasicDBObject("status", 0));
|
||||
//int count = 0;
|
||||
//while(cursor_artemis_message.hasNext()){
|
||||
|
||||
+165
-136
@@ -1,8 +1,18 @@
|
||||
package com.fangjia.script.dataAna
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.common.collect.Lists
|
||||
import com.google.common.collect.Sets
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.Semaphore
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.google.common.collect.Lists
|
||||
import com.google.common.collect.Sets
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
|
||||
@BatchOwner(Watcher.LiXue)
|
||||
class CityInfoAnaBatch extends BaseBatch {
|
||||
|
||||
@@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
|
||||
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
|
||||
final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo);
|
||||
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
|
||||
final static DSCollection SEAWEED = DbUtils.seaweed
|
||||
// final static DSCollection SEAWEED = DbUtils.seaweed
|
||||
static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
final static def CITYS = [];
|
||||
@@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
|
||||
|
||||
allCitys.each {k,v ->
|
||||
// if(["南京"].contains(k.toString())){
|
||||
// if(["北京"].contains(k.toString())){
|
||||
CITYS << k.toString();
|
||||
// }
|
||||
}
|
||||
@@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
|
||||
// 城市小区数
|
||||
def allCityDisCount = [:];
|
||||
LinkedHashMap<String, Object> facet = new LinkedHashMap<String, Object>(2);
|
||||
facet.put("fields", ["city"]);
|
||||
facet.put('limit', 500);
|
||||
|
||||
DBObject disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'district');
|
||||
|
||||
DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
List<FacetResult> disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityDisCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district"));
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
groupFields.put("_id", new BasicDBObject("city", '$city'))
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityDisCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
// 城市新盘数
|
||||
def allCityNewDisCount = [:];
|
||||
disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'new_district');
|
||||
|
||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityNewDisCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district"));
|
||||
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityNewDisCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
// 城市写字楼数
|
||||
def allCityOfficeCount = [:];
|
||||
disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'office');
|
||||
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office"));
|
||||
|
||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityOfficeCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityOfficeCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
|
||||
// 城市商铺数
|
||||
def allCityShopCount = [:];
|
||||
disQuery = new BasicDBObject();
|
||||
disQuery.put('$facet', facet);
|
||||
disQuery.put("visible", 0);
|
||||
disQuery.put("cat", 'shop');
|
||||
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop"));
|
||||
|
||||
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
|
||||
disFacetResult = sellDSCursor.getFacetResults();
|
||||
if (disFacetResult) {
|
||||
def obj = disFacetResult[0].values;
|
||||
obj.each {
|
||||
allCityShopCount.put(it.name, it.count.intValue());
|
||||
}
|
||||
output = SEAWEED.aggregate(Arrays.asList(match, group));
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
allCityShopCount.put(obj.get("city"), result.get("count"))
|
||||
}
|
||||
|
||||
// 城市均价
|
||||
def allCityStats = [:];
|
||||
query = new BasicDBObject();
|
||||
@@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
|
||||
}
|
||||
|
||||
//统计有楼栋的小区数和有房号的小区数
|
||||
// countDistrictsHasLoudong(allCitys, weekend)
|
||||
//countDistrictsHasLoudongNew(allCitys, weekend)
|
||||
|
||||
// //统计有楼栋的小区数和有房号的小区数
|
||||
//// countDistrictsHasLoudong(allCitys, weekend)
|
||||
// //countDistrictsHasLoudongNew(allCitys, weekend)
|
||||
//
|
||||
final CountDownLatch latch = new CountDownLatch(CITYS.size())
|
||||
final Semaphore semaphore = new Semaphore(10)
|
||||
CITYS.each {
|
||||
@@ -239,13 +202,13 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
} finally {
|
||||
semaphore.release()
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t楼栋覆盖率统计完成"
|
||||
|
||||
|
||||
latch = new CountDownLatch(CITYS.size())
|
||||
semaphore = new Semaphore(10)
|
||||
CITYS.each {
|
||||
@@ -267,12 +230,12 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
}
|
||||
latch.await()
|
||||
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t房号覆盖率统计完成"
|
||||
|
||||
|
||||
|
||||
|
||||
//71个城市的覆盖率统计
|
||||
sendCountData(weekend)
|
||||
|
||||
|
||||
|
||||
|
||||
//最后统计之前因为太慢取消统计的本周估价次数
|
||||
latch = new CountDownLatch(CITYS.size())
|
||||
semaphore = new Semaphore(10)
|
||||
@@ -292,14 +255,14 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
query.put('city', cityName);
|
||||
query.put('c_date', new BasicDBObject('$gte', laseWeekend).append('$lte', weekend));
|
||||
entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));
|
||||
|
||||
|
||||
if (entity.get('evaluation_count')) {
|
||||
entity.put('price_to_eva', DataUtil.roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
|
||||
}
|
||||
CITY_INFO_CONN.save(entity)
|
||||
println cityName+"本周估价次数统计完毕"
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
println e.getMessage()
|
||||
} finally {
|
||||
@@ -343,21 +306,21 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
def totalDbhouseloudongCount = HOUSE_LOUDONG_CONN.count(findObj)
|
||||
datas << ['', totalDistrictCount, totalLoudongCount, df.format(totalLoudongCount * 1.0/ totalDistrictCount), totalFanghaoCount, df.format(totalFanghaoCount * 1.0/ totalDistrictCount), totalDbLoudongCount, totalDbhouseloudongCount]
|
||||
|
||||
ExcelUtils.write(fileName, datas, resultTitle)
|
||||
String id = ObjectId.get().toString()
|
||||
BasicDBObject file = new BasicDBObject()
|
||||
file.put("t", "upload")
|
||||
file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
||||
file.put("c_date", new Date())
|
||||
file.put("_id", idGenerator(id, "o"))
|
||||
file.put("data", new File(fileName).getBytes())
|
||||
fsDB.save(file)
|
||||
def href = DataUtil.encodePicUrl(id, "o", "upload")
|
||||
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
|
||||
receivers.each {receiver->
|
||||
jms.sendMail(receiver, resultTitle, href)
|
||||
}
|
||||
new File(fileName).deleteOnExit()
|
||||
// ExcelUtils.write(fileName, datas, resultTitle)
|
||||
// String id = ObjectId.get().toString()
|
||||
// BasicDBObject file = new BasicDBObject()
|
||||
// file.put("t", "upload")
|
||||
// file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
||||
// file.put("c_date", new Date())
|
||||
// file.put("_id", idGenerator(id, "o"))
|
||||
// file.put("data", new File(fileName).getBytes())
|
||||
// fsDB.save(file)
|
||||
// def href = DataUtil.encodePicUrl(id, "o", "upload")
|
||||
// href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
|
||||
// receivers.each {receiver->
|
||||
// jms.sendMail(receiver, resultTitle, href)
|
||||
// }
|
||||
// new File(fileName).deleteOnExit()
|
||||
}
|
||||
|
||||
private String idGenerator(String url, String size) {
|
||||
@@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
|
||||
|
||||
|
||||
// if (allCityNewDisCount.get(cityName)) {
|
||||
// def newDisCount = allCityNewDisCount.get(cityName);
|
||||
// if (allCityNewDisCount.get(cityName)) {
|
||||
// def newDisCount = allCityNewDisCount.get(cityName);
|
||||
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
|
||||
// } else {
|
||||
// entity.put('new_district_detail', [:]);
|
||||
@@ -604,23 +567,30 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
if (officePrices) {
|
||||
detail.put('price', roundDouble(officePrices.size(), officeCount));
|
||||
}
|
||||
|
||||
query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('kind', '写字楼');
|
||||
query.put('weekend', weekend);
|
||||
def officeRents = RENT_STAT_CONN.distinct("name", query);
|
||||
|
||||
def officeRents = findOfficeRent(cityName)
|
||||
if (officeRents) {
|
||||
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
|
||||
}
|
||||
|
||||
def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName)
|
||||
if (halfYearofficeRentCount) {
|
||||
detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount));
|
||||
}
|
||||
|
||||
|
||||
entity.put('office_detail', detail);
|
||||
entity.put('existing_office_count', officeCount);
|
||||
} else {
|
||||
entity.put('office_detail', [:]);
|
||||
}
|
||||
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
|
||||
|
||||
// if(curInfo){
|
||||
// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity))
|
||||
// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
|
||||
// }
|
||||
|
||||
|
||||
if (entity.get('official_population') && entity.get('district_avg_population')) {
|
||||
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
|
||||
|
||||
@@ -628,12 +598,12 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
entity.put('district_coverage', roundDouble(entity.get('existing_district_count'), entity.get('prediction_district_count')));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
if (allCityShopCount.get(cityName)) {
|
||||
def shopCount = allCityShopCount.get(cityName);
|
||||
def detail = shopDetail(shopCount, cityName);
|
||||
|
||||
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
//query.put('estate_type', '商铺');
|
||||
@@ -642,7 +612,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
if (shopPrices) {
|
||||
detail.put('price', roundDouble(shopPrices.size(), shopCount));
|
||||
}
|
||||
|
||||
|
||||
query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('kind', '商铺');
|
||||
@@ -651,45 +621,41 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
if (shopRents) {
|
||||
detail.put('rentPrice', roundDouble(shopRents.size(), shopCount));
|
||||
}
|
||||
|
||||
|
||||
entity.put('shop_detail', detail);
|
||||
entity.put('existing_shop_count', shopCount);
|
||||
}else {
|
||||
entity.put('shop_detail', [:]);
|
||||
}
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 商铺完成]"
|
||||
|
||||
|
||||
def cityStat = allCityStats.get(cityName);
|
||||
if (cityStat) {
|
||||
entity.put('avg_price', cityStat.avg_price);
|
||||
}
|
||||
|
||||
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('c_date', new BasicDBObject('$gte', lastWeekend).append('$lte', weekend));
|
||||
//entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
|
||||
entity.put('price_total_count', (int) PRICE_JIUCUO_CONN.count(query));
|
||||
|
||||
|
||||
query.put('price_lev', '偏高');
|
||||
entity.put('price_high_count', (int) PRICE_JIUCUO_CONN.count(query));
|
||||
|
||||
|
||||
query.put('price_lev', '偏低');
|
||||
entity.put('price_low_count', (int) PRICE_JIUCUO_CONN.count(query));
|
||||
|
||||
|
||||
/*if (entity.get('evaluation_count')) {由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
|
||||
entity.put('price_to_eva', roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
|
||||
}*/
|
||||
|
||||
|
||||
if (entity.get('price_high_count')) {
|
||||
entity.put('low_to_high', roundDouble(entity.get('price_low_count'), entity.get('price_high_count')));
|
||||
}
|
||||
|
||||
|
||||
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// query = new BasicDBObject();
|
||||
// query.put('city', cityName);
|
||||
// query.put('weekend', weekend);
|
||||
@@ -710,7 +676,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if (total > 0) {
|
||||
// BigDecimal bg = new BigDecimal(webZhunque * 1.0 / total);
|
||||
// entity.put('web_average', bg.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue());
|
||||
@@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
fields.put(key, 1)
|
||||
}
|
||||
|
||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
def cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
def countMap = [:]
|
||||
println cur.count()
|
||||
@@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
fields.put(key, 1)
|
||||
}
|
||||
|
||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
def cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
def countMap = [:]
|
||||
|
||||
@@ -1546,6 +1512,69 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
def findOfficeRent(cityName){
|
||||
def query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
// query.put('kind', '写字楼');
|
||||
// query.put('weekend', weekend);
|
||||
// def officeRents = RENT_STAT_CONN.distinct("name", query);
|
||||
// if (officeRents) {
|
||||
// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
|
||||
// }
|
||||
query.put('cat', 'office');
|
||||
def date1 = new Date()
|
||||
date1.setDate(28)
|
||||
if(new Date().getTime()> date1.getTime()){
|
||||
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")));
|
||||
}else{
|
||||
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")));
|
||||
}
|
||||
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
|
||||
return officeRents
|
||||
}
|
||||
|
||||
def findHalfYearOfficeRentCount(cityName){
|
||||
|
||||
def date1 = new Date()
|
||||
date1.setDate(28)
|
||||
def months = []
|
||||
if(new Date().getTime()> date1.getTime()){
|
||||
for(def i=1;i<=6;i++){
|
||||
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
|
||||
}
|
||||
}else{
|
||||
for(def i=2;i<=7;i++){
|
||||
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
|
||||
}
|
||||
}
|
||||
def map = [:]
|
||||
for(def month : months){
|
||||
def query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('cat', 'office');
|
||||
query.put('date', month);
|
||||
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
|
||||
|
||||
officeRents.each{k->
|
||||
def count = map.get(k)
|
||||
if(!count){
|
||||
map.put(k, 1)
|
||||
}else{
|
||||
map.put(k, count + 1)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def total = 0
|
||||
map.each{k,v->
|
||||
if(v==6){
|
||||
total ++
|
||||
}
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -21,8 +21,8 @@ import java.util.concurrent.TimeUnit
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictTradeHouseStatBatch extends BaseBatch{
|
||||
|
||||
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "192.168.0.43")
|
||||
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictTradeHouseStatBatch(), args)
|
||||
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
package com.fangjia.script.dealPrice
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
/**
|
||||
* 成交同步206,按日同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class SynchDealPriceToEs extends BaseBatch{
|
||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDealPriceToEs(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
|
||||
def q = new BasicDBObject()
|
||||
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcursor = dealPriceConn.find(q)
|
||||
println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
|
||||
def ret = []
|
||||
for(def o : dbcursor){
|
||||
ret << o
|
||||
}
|
||||
save(ret)
|
||||
|
||||
// def batchsize = 2000
|
||||
// def citys = getCityCount()
|
||||
// citys.each{city,count->
|
||||
// if(city instanceof String){
|
||||
// for(def year = 1999;year<=2025;year++){
|
||||
// def q = new BasicDBObject("city", city)
|
||||
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
|
||||
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
|
||||
// println city +" "+ dbcursor.count()
|
||||
// def ret = []
|
||||
// for(def o : dbcursor){
|
||||
// ret << o
|
||||
// }
|
||||
// save(ret)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
def getCityCount(){
|
||||
|
||||
BasicDBObject match = null;
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
|
||||
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
|
||||
groupFields.put("_id", new BasicDBObject("city",'$city'))
|
||||
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
|
||||
// 执行聚合查询
|
||||
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
|
||||
def map = [:]
|
||||
// 遍历结果
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
if(obj.get("city")) {
|
||||
int count = (Integer) result.get("count");
|
||||
map.put(obj.get("city"), count)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 500;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.206")
|
||||
|
||||
for(def map : rows){
|
||||
// def map = rows.get(key)
|
||||
def data_id = map._id.toString()
|
||||
map.remove("_id")
|
||||
request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+187
-68
@@ -1,6 +1,5 @@
|
||||
package com.fangjia.script.district
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
@@ -9,11 +8,20 @@ import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
/**
|
||||
* Created by dxy on 2016/12/15.
|
||||
*/
|
||||
@@ -26,6 +34,10 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
private static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
private static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
|
||||
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
|
||||
public void process(Object args) {
|
||||
def cities = [
|
||||
@@ -47,7 +59,7 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
"图木舒克","五家渠","济源","仙桃","潜江","天门","景德镇", "神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水",
|
||||
"保亭","琼中","北屯","铁门关","双河","可克达拉"
|
||||
]
|
||||
cities = ["ÎÞÎý"]
|
||||
cities = ["鹤岗"]
|
||||
println cities
|
||||
for(String city : cities) {
|
||||
List<DBObject> districts = findDistrictByCity(city)
|
||||
@@ -63,14 +75,14 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
for(DBObject district : districts) {
|
||||
String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city")
|
||||
long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0
|
||||
if(district.get('alias')) {
|
||||
for (String ali : district.get('alias')) {
|
||||
if(ali) {
|
||||
String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
|
||||
sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(district.get('alias')) {
|
||||
// for (String ali : district.get('alias')) {
|
||||
// if(ali) {
|
||||
// String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
|
||||
// sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
sellCountsList << (int)sellCount
|
||||
}
|
||||
sellCountsList = sellCountsList.unique()
|
||||
@@ -219,67 +231,74 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
private def findDistrictHouseCount(String city, List<DBObject> districts, Date date) {
|
||||
def regionDistrictMap = [:]
|
||||
for(DBObject dbo : districts) {
|
||||
def names = []
|
||||
String n = DataUtil.escapeSolr(dbo.get('name'))
|
||||
if(n) {
|
||||
names << n
|
||||
}
|
||||
if(dbo.get("alias")) {
|
||||
for(String ali : dbo.get("alias")) {
|
||||
ali = DataUtil.escapeSolr(ali)
|
||||
if(ali) {
|
||||
names << ali
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String region = dbo.get('region')
|
||||
if(!regionDistrictMap.containsKey(region)) {
|
||||
regionDistrictMap.put(region, names)
|
||||
}else {
|
||||
def list = regionDistrictMap.get(region)
|
||||
list.addAll(names)
|
||||
regionDistrictMap.put(region, list)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -1)),"yyyyMMdd")
|
||||
def endDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(date),"yyyyMMdd")
|
||||
Map<String, Long> monthMap = findAllCommHouse(city, districts, startDate.toInteger(), endDate.toInteger())
|
||||
|
||||
Map<String, Long> halfMap = new HashMap<String, Long>();
|
||||
Map<String, Long> monthMap = new HashMap<String, Long>();
|
||||
for(String region : regionDistrictMap.keySet()) {
|
||||
def nameGroups = []
|
||||
int len = 0
|
||||
def nameList = []
|
||||
for(String name : regionDistrictMap.get(region)) {
|
||||
if(len >= 600 || nameList.size() >= 100) {
|
||||
nameGroups << new ArrayList(nameList);
|
||||
nameList.clear()
|
||||
len = 0
|
||||
}
|
||||
len += name.length()
|
||||
nameList << name
|
||||
}
|
||||
if(nameList.size() > 0) {
|
||||
nameGroups << new ArrayList(nameList);
|
||||
}
|
||||
def startDate2 = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -6)),"yyyyMMdd")
|
||||
Map<String, Long> halfMap = findAllCommHouse(city, districts, startDate2.toInteger(), endDate.toInteger())
|
||||
|
||||
println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
|
||||
for(List<String> names : nameGroups) {
|
||||
def halfRegionResult = [:]
|
||||
def monthRegionResult = [:]
|
||||
try {
|
||||
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
}catch (e) {
|
||||
sleep(10 * 1000)
|
||||
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
}
|
||||
halfMap.putAll(halfRegionResult)
|
||||
monthMap.putAll(monthRegionResult)
|
||||
}
|
||||
}
|
||||
// def regionDistrictMap = [:]
|
||||
// for(DBObject dbo : districts) {
|
||||
// def names = []
|
||||
// String n = DataUtil.escapeSolr(dbo.get('name'))
|
||||
// if(n) {
|
||||
// names << n
|
||||
// }
|
||||
// if(dbo.get("alias")) {
|
||||
// for(String ali : dbo.get("alias")) {
|
||||
// ali = DataUtil.escapeSolr(ali)
|
||||
// if(ali) {
|
||||
// names << ali
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// String region = dbo.get('region')
|
||||
// if(!regionDistrictMap.containsKey(region)) {
|
||||
// regionDistrictMap.put(region, names)
|
||||
// }else {
|
||||
// def list = regionDistrictMap.get(region)
|
||||
// list.addAll(names)
|
||||
// regionDistrictMap.put(region, list)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// for(String region : regionDistrictMap.keySet()) {
|
||||
// def nameGroups = []
|
||||
// int len = 0
|
||||
// def nameList = []
|
||||
// for(String name : regionDistrictMap.get(region)) {
|
||||
// if(len >= 600 || nameList.size() >= 100) {
|
||||
// nameGroups << new ArrayList(nameList);
|
||||
// nameList.clear()
|
||||
// len = 0
|
||||
// }
|
||||
// len += name.length()
|
||||
// nameList << name
|
||||
// }
|
||||
// if(nameList.size() > 0) {
|
||||
// nameGroups << new ArrayList(nameList);
|
||||
// }
|
||||
//
|
||||
// println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
|
||||
// for(List<String> names : nameGroups) {
|
||||
// def halfRegionResult = [:]
|
||||
// def monthRegionResult = [:]
|
||||
// try {
|
||||
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
// }catch (e) {
|
||||
// sleep(10 * 1000)
|
||||
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
|
||||
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
|
||||
// }
|
||||
// halfMap.putAll(halfRegionResult)
|
||||
// monthMap.putAll(monthRegionResult)
|
||||
// }
|
||||
// }
|
||||
|
||||
return [halfMap, monthMap];
|
||||
}
|
||||
@@ -352,6 +371,106 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
|
||||
return ret
|
||||
}
|
||||
|
||||
def findAllCommHouse(city, districts, startDate, endDate){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = findHousesByDistrict(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(count){
|
||||
String key = rMap.get("name") + "_" + rMap.get("region") + "_" + rMap.get("city")
|
||||
retMap.put(key, count)
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+count
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
def count = 0
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
count = findHousesByMdate(city, region, cat, startDate, endDate, name, alias)
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(3000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictDealSellRentRankBatch(), args)
|
||||
}
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ class AbcEvaluationRowsBatch extends BaseBatch{
|
||||
|
||||
static final def fjtSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
|
||||
def charset = "GBK"
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ class FzDataResultStat {
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
|
||||
@@ -169,7 +169,7 @@ DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
//
|
||||
//
|
||||
//def count(name, sDate){
|
||||
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city","¸£ÖÝ")
|
||||
// q.put("s_date", sDate)
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import java.util.concurrent.CountDownLatch
|
||||
|
||||
class ThirdTest {
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class TradeExportBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
static def districtMap = [:]
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class CzBankStat {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "mongo.db.fangjia.com")
|
||||
def static final DBCollection CZLOG = MongoUtils.getCollectionByName("fangjia_log", "cz_log2", "192.168.0.136")
|
||||
static main(args) {
|
||||
def task = new CzBankStat();
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ import org.apache.commons.lang.StringUtils
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DealPriceProcess {
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
def total = 0
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ import java.util.concurrent.Executors
|
||||
|
||||
class DistrictDetailBatch {
|
||||
|
||||
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "192.168.0.43")
|
||||
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ class EsHouseStat {
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
public static void main(String[] args){
|
||||
|
||||
@@ -16,8 +16,8 @@ import java.sql.ResultSet
|
||||
|
||||
class FZDataExport {
|
||||
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class FZHouseEvaluation {
|
||||
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
|
||||
|
||||
def regionMap = [:]
|
||||
|
||||
+59
-2
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import com.mongodb.DBCollection;
|
||||
@@ -12,12 +13,13 @@ import com.fangjia.script.util.ExcelUtils;
|
||||
class FangjiatongUserStat {
|
||||
|
||||
public static final DBCollection FJT_USER_COLL = MongoUtils.getCollectionByName("fangjiatong_log", "utm_gif_log", "222.73.242.42", 52002)
|
||||
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
def task = new FangjiatongUserStat();
|
||||
task.run();
|
||||
// task.run();
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -60,4 +62,59 @@ class FangjiatongUserStat {
|
||||
return ret;
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def users = findUses(12)
|
||||
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
|
||||
def rets = []
|
||||
def header = ["用户id","支行名称", "备注名称"]
|
||||
months.each{m->
|
||||
header << m.substring(0, 7)
|
||||
}
|
||||
rets << header
|
||||
for(def m : users){
|
||||
def ret = []
|
||||
ret << m.user_id
|
||||
ret << m.name
|
||||
ret << m.org_name
|
||||
for(def i=1;i<months.size();i++){
|
||||
def evalCount = findEvaluationCount(m.user_id, months[i-1], months[i])
|
||||
ret << evalCount
|
||||
}
|
||||
|
||||
for(def i=1;i<months.size();i++){
|
||||
def downCount = findTradesCount(m.user_id, months[i-1], months[i])
|
||||
ret << downCount
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019")
|
||||
}
|
||||
|
||||
def findUses(role){
|
||||
def sql = "select u.id,ud.name,ud.org_name from (SELECT * from users WHERE role=12) u left join users_detail ud on u.id=ud.id ";
|
||||
def rets = [];
|
||||
onLineSqlInstance.eachRow(sql, {
|
||||
rets << ["user_id":it.id, "name":it.name, "org_name":it.org_name]
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
def findEvaluationCount(user_id, startDate, endDate){
|
||||
def sql = "select count(*) as count from evaluation where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
|
||||
def count = 0;
|
||||
onLineSqlInstance.eachRow(sql, {
|
||||
count = it.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findTradesCount(user_id, startDate, endDate){
|
||||
def sql = "select count(*) as count from user_trades where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
|
||||
def count = 0;
|
||||
onLineSqlInstance.eachRow(sql, {
|
||||
count = it.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -22,9 +22,6 @@ class FiledStrand {
|
||||
|
||||
def updateIndex = 0;
|
||||
static main(args) {
|
||||
|
||||
|
||||
|
||||
// def task = new FiledStrand();
|
||||
// task.run();
|
||||
// task.updateOpenDate();
|
||||
@@ -271,7 +268,7 @@ class FiledStrand {
|
||||
static def processArea(all_area){
|
||||
if(all_area){
|
||||
def v = getValue(all_area)
|
||||
if(v>=5000 && v<=10000000){
|
||||
if(v>=1000 && v<=10000000){
|
||||
return v.toString().toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
@@ -288,11 +285,25 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def processLobbyHeight(height){
|
||||
if(height){
|
||||
def v = getValue(height)
|
||||
if(v>=2.4 && v<=30){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processElevators(elevators){
|
||||
if(elevators){
|
||||
def v = getValue(elevators)
|
||||
if(v>0 && v<=100){
|
||||
return v.toString()
|
||||
if(v instanceof Integer){
|
||||
if(v>0 && v<=100){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
@@ -317,24 +328,34 @@ class FiledStrand {
|
||||
|
||||
static def processStandardArea(area){
|
||||
if(area){
|
||||
area = area.toString().replace("£¬",",").replace("©O", "").replace("m", "").replace("/","")
|
||||
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
|
||||
return area
|
||||
def v = getValue(area)
|
||||
if(v>=500 && v<=20000){
|
||||
return v.toString().toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processCarSiteRent(rent){
|
||||
if(rent){
|
||||
def v = getNumberRange(rent)
|
||||
if(v){
|
||||
if(v=="0"){
|
||||
if(["-","0元/月","无","不详","未知","待查","免费元/月","/","0","*","待确认","undefined元/月","0元","0元每月","未知元/月","0.00元/月","未定","暂定","测算中","无元/月","未定元/月","另议元/月","面议元/月","面议元/月","有元/月","暂时未确定","0元(无停车位)"," 元/月"].contains(rent)){
|
||||
return null
|
||||
}else if(rent.contains("暂无") || rent.contains("车位不归") || rent.contains("待定") || rent.contains("暂定") || rent.contains("暂完善") || rent.contains("待审批") || rent.contains("不详") || rent.contains("未定价格") || rent.contains("中国电信") || rent.contains("面谈")){
|
||||
return null
|
||||
}else{
|
||||
rent = rent.replace("元/月","元/月").replace(",","").replace("#","").replace(".00","")
|
||||
if(rent.toString().length()>100){
|
||||
return null
|
||||
}else{
|
||||
return v.toString()
|
||||
}
|
||||
if(rent instanceof Double && rent.toDouble()<100){
|
||||
return null
|
||||
}
|
||||
|
||||
if(rent instanceof Integer && rent.toInteger()<100){
|
||||
return null
|
||||
}
|
||||
return rent
|
||||
}
|
||||
return null
|
||||
}
|
||||
return null
|
||||
}
|
||||
@@ -382,7 +403,7 @@ class FiledStrand {
|
||||
static def processCarSite(car_site_count){
|
||||
def v = getValue(car_site_count.toString());
|
||||
if(v){
|
||||
if(v>5 && v<=100000){
|
||||
if(v>5 && v<=10000){
|
||||
try {
|
||||
return v.toString().toInteger();
|
||||
}catch(Exception e){
|
||||
@@ -533,12 +554,82 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processLevel(val){
|
||||
def level = null
|
||||
if (val) {
|
||||
if(val.contains("超甲") || val.contains("5A") || val.contains("6A") || val.contains("顶级")){
|
||||
level = "超甲级"
|
||||
}else if(val.indexOf("甲")>-1 || val.contains("A")){
|
||||
level = "甲级"
|
||||
}else if(val.indexOf("乙")>-1 || val.contains("B")){
|
||||
level = "乙级";
|
||||
}else if(val.indexOf("丙")>-1 || val.contains("C")){
|
||||
level = "丙级";
|
||||
}
|
||||
}
|
||||
return level
|
||||
}
|
||||
|
||||
|
||||
static def processLoopLine(val){
|
||||
def line = null
|
||||
if (val) {
|
||||
if(!val.contains("暂") && !val.contains("无")){
|
||||
if(val.indexOf("环")>-1){
|
||||
line = val
|
||||
}
|
||||
}
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
static def processForeign(val){
|
||||
def foreign = null
|
||||
if (val) {
|
||||
if(val.indexOf("是")>-1){
|
||||
foreign = "是"
|
||||
}else if(val.indexOf("否")>-1){
|
||||
foreign = "否";
|
||||
}
|
||||
}
|
||||
return foreign
|
||||
}
|
||||
|
||||
static def processRegister(val){
|
||||
def register = null
|
||||
if (val) {
|
||||
if(val.contains("不可以注册") || val.contains("不能注册")){
|
||||
register = "否";
|
||||
}else if(val.contains("可以注册")){
|
||||
register = "是"
|
||||
}else if(val.indexOf("是")>-1){
|
||||
register = "是"
|
||||
}else if(val.indexOf("否")>-1){
|
||||
register = "否";
|
||||
}
|
||||
}
|
||||
return register
|
||||
}
|
||||
|
||||
static def processFitment(val){
|
||||
def fitment = null
|
||||
if(val){
|
||||
if(val.indexOf("精装")>-1){
|
||||
fitment = "精装"
|
||||
}else if(val.indexOf("简单")>-1 || val.indexOf("简装")>-1 || val.indexOf("一般")>-1 || val.indexOf("普通")>-1){
|
||||
fitment = "简装";
|
||||
}else if(val.indexOf("毛坯")>-1){
|
||||
fitment = "毛坯";
|
||||
}
|
||||
}
|
||||
return fitment
|
||||
}
|
||||
|
||||
static def processLngAndLat(val) {
|
||||
if (val) {
|
||||
if (val && val!="null") {
|
||||
def map = [:]
|
||||
def lng = DataHandleUtils.getDoubleRound(6, val.x)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, val.y)
|
||||
def lng = DataHandleUtils.getDoubleRound(6, val.lon)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, val.lat)
|
||||
Long gd_lat = lat * 10000000000L
|
||||
Long gd_lng = lng * 10000000000L
|
||||
map.lat = gd_lat
|
||||
@@ -618,7 +709,8 @@ class FiledStrand {
|
||||
if(cubage_percent){
|
||||
def v = getValue(cubage_percent)
|
||||
if(v){
|
||||
if(v>=0.3 && v<=10){
|
||||
v = DataHandleUtils.getDoubleRound(2, v.toString().toDouble())
|
||||
if(v>=0.3 && v<=20){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
@@ -644,7 +736,7 @@ class FiledStrand {
|
||||
static def processVirescencePercent(virescence_percent){
|
||||
if(virescence_percent){
|
||||
def v = getValue(virescence_percent)
|
||||
if(v && v>15 && v<=70){
|
||||
if(v && v>=10 && v<=70){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
@@ -788,6 +880,20 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processUndergroundFloor(floor){
|
||||
def v = getValue(floor.toString());
|
||||
if(v){
|
||||
if(v>=1 && v<=5){
|
||||
try {
|
||||
return v.toString();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processRealUsePercent(percent){
|
||||
def v = getValue(percent.toString());
|
||||
if(v){
|
||||
@@ -834,15 +940,40 @@ class FiledStrand {
|
||||
|
||||
static def processCompany(val){
|
||||
if(val){
|
||||
if(val.length()>20){
|
||||
if(val.length()>20 || val.length()<4){
|
||||
return null
|
||||
}else{
|
||||
if(val.contains("ÔÝÎÞ")){
|
||||
if(["#NA","*"].contains(val)){
|
||||
return null
|
||||
}
|
||||
if(val.contains("暂无") || val.contains("暂定")){
|
||||
return null
|
||||
}
|
||||
if(val.contains("未知") || val.contains("不祥")){
|
||||
return null
|
||||
}
|
||||
if(val.contains("待定") || val.contains("待详情")){
|
||||
return null
|
||||
}
|
||||
if(val.contains("尚未公开")){
|
||||
return null
|
||||
}
|
||||
if(val.contains("在线咨询")){
|
||||
return null
|
||||
}
|
||||
|
||||
if(val.contains("平米") || val.contains("元平月")){
|
||||
return null
|
||||
}
|
||||
if(val == "ÎÞ"){
|
||||
return null
|
||||
}
|
||||
if(val == "-"){
|
||||
return null
|
||||
}
|
||||
if(val == "--"){
|
||||
return null
|
||||
}
|
||||
if(val.isNumber()){
|
||||
return null
|
||||
}
|
||||
@@ -863,7 +994,7 @@ class FiledStrand {
|
||||
if (pattern.matcher(val).matches()) {
|
||||
return null
|
||||
}
|
||||
return val
|
||||
return val.replace("*","").replace("*","")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -909,14 +1040,18 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
static def getNumber(def input) {
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
try{
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
}
|
||||
}catch(Exception e){
|
||||
println input+" error"
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class FjwUserStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
|
||||
public static final DBCollection USER_COLL = MongoUtils.getCollectionByName2("fangjia_coresite", "user", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import org.bson.Document;
|
||||
import org.bson.conversions.Bson;
|
||||
|
||||
class HouseLouDongMatch {
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ class LandDetailCrawl {
|
||||
|
||||
DBCollection LAND_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "land_deal_price1", "192.168.0.235", 27018)
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", lineUrl)
|
||||
public static final DBCollection OFF_FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", offlineUrl)
|
||||
|
||||
@@ -14,10 +14,10 @@ import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class LogStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
|
||||
+2
-2
@@ -21,8 +21,8 @@ import java.util.concurrent.Executors
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class NoHouseDistrictStatBatch extends BaseBatch{
|
||||
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
|
||||
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ class PingAnAddressLevalTest {
|
||||
|
||||
String token = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB";
|
||||
|
||||
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
|
||||
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new PingAnAddressLevalTest()
|
||||
task.run();
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
|
||||
class ProxyHttpUtils {
|
||||
|
||||
static String proxyHost = 'tunnel2.qg.net';
|
||||
static Integer proxyPort = 14186;
|
||||
static String proxyUsername = '113BBC06';
|
||||
static String proxyPassword = '3B2FBD05B0AF';
|
||||
|
||||
static String getProxyResult(targetUrl, data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
String jsonData = new JsonOutput().toJson(data);
|
||||
// 创建代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
|
||||
// 创建 URL 对象
|
||||
URL url = new URL(targetUrl);
|
||||
|
||||
// 创建连接对象
|
||||
URLConnection connection = url.openConnection(proxy);
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST");
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true);
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json");
|
||||
|
||||
// 发送请求参数
|
||||
OutputStream outputStream = connection.getOutputStream();
|
||||
outputStream.write(jsonData.getBytes("UTF-8"));
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 读取响应
|
||||
int responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line);
|
||||
}
|
||||
reader.close();
|
||||
return response.toString();
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -329,7 +329,7 @@ def findDistrictPrice(def city, def region){
|
||||
q.put("region", region);
|
||||
}
|
||||
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
|
||||
def count = districtStatConn.count(q)
|
||||
return count;
|
||||
}
|
||||
@@ -343,7 +343,7 @@ def findDistrictPrice(def city, def region){
|
||||
q.put("region", region);
|
||||
}
|
||||
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
|
||||
def count = districtStatConn.count(q)
|
||||
return count;
|
||||
}
|
||||
+1
-1
@@ -6,7 +6,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class RegionBlockSetting {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName2("fangjia", "region_block", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+2
-2
@@ -17,8 +17,8 @@ import com.mongodb.DBObject
|
||||
class SynchDistrictStatBatch extends BaseBatch{
|
||||
|
||||
|
||||
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class UpdateThirdPrice {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col();
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ import groovy.sql.Sql
|
||||
class ApiUserCountAnalyBatch extends BaseBatch{
|
||||
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
|
||||
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class CcbCityPriceStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
@@ -104,6 +104,43 @@ class FileUtils {
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
return sb.toString().getBytes(charset)
|
||||
}
|
||||
|
||||
static def createEncryptFile(fileName, outputStr, isAppend){
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
out.write(outputStr.getBytes("utf-8"));
|
||||
out.close();
|
||||
}
|
||||
|
||||
|
||||
static def createFileStringByCustomSuffix(ret, fileName, charset, suffix, lineSuffix){
|
||||
File file = new File(fileName);
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
def sb = new StringBuffer();
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), suffix))
|
||||
if(index<ret.size()){
|
||||
sb.append(lineSuffix+"\n");
|
||||
}else{
|
||||
sb.append(lineSuffix);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return sb.toString()
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ import com.mongodb.DBCollection
|
||||
*/
|
||||
class PocTest2023 {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ import org.apache.commons.lang.math.RandomUtils
|
||||
class DistrictSelectHouse {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSelectHouse();
|
||||
|
||||
+2
-2
@@ -9,8 +9,8 @@ import com.mongodb.QueryOperators
|
||||
|
||||
class TestPrice {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class CityStatExport {
|
||||
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class CityStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||
|
||||
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
|
||||
|
||||
+3
-3
@@ -21,13 +21,13 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
//def citys = ["ÎÚ³ľÆë","²ý¼ª","°¢¿ËËÕ","ÒÁÀç","¿¦Ê²"]
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.gj.coresite
|
||||
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import java.io.OutputStream
|
||||
|
||||
// 目标服务器的URL
|
||||
def targetUrlString = "http://data.zz.baidu.com/urls?site=sh.fangjia.com&token=UaKTemkcQ6I4YvSA"
|
||||
|
||||
// 需要POST的URL数据
|
||||
def postData = "https://sh.fangjia.com/zoushi?cityName=%E4%B8%8A%E6%B5%B7&keyword=%E5%A4%A9%E5%B1%B1%E6%96%B0%E8%8B%91"
|
||||
|
||||
// 创建URL对象
|
||||
URL url = new URL(targetUrlString)
|
||||
|
||||
// 打开连接
|
||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection()
|
||||
connection.setDoOutput(true) // 表明我们将发送POST请求的输出数据
|
||||
connection.setRequestMethod("POST") // 设置请求方法为POST
|
||||
connection.setRequestProperty("User-Agent", "curl/7.12.1") // 设置User-Agent,尽管这在实际应用中可能并非必需
|
||||
connection.setRequestProperty("Content-Type", "text/plain") // 设置内容类型,因为我们将发送纯文本数据
|
||||
connection.setRequestProperty("Content-Length", Integer.toString(postData.getBytes().length)) // 设置内容长度
|
||||
|
||||
// 发送POST数据
|
||||
//try (OutputStream os = connection.getOutputStream()) {
|
||||
// byte[] input = postData.getBytes("utf-8")
|
||||
// os.write(input, 0, input.length)
|
||||
//}
|
||||
outputStream = connection.getOutputStream();
|
||||
outputStream.write(postData.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 获取响应码
|
||||
int responseCode = connection.getResponseCode()
|
||||
println "POST Response Code :: $responseCode"
|
||||
|
||||
// 可选:读取响应内容
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"))
|
||||
StringBuilder response = new StringBuilder()
|
||||
String responseLine
|
||||
while ((responseLine = reader.readLine()) != null) {
|
||||
response.append(responseLine.trim())
|
||||
}
|
||||
println "Response Body :: ${response.toString()}"
|
||||
|
||||
// 关闭连接
|
||||
connection.disconnect()
|
||||
+1
-1
@@ -11,7 +11,7 @@ import com.mongodb.QueryOperators
|
||||
class BestDistrictPriceExport {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new BestDistrictPriceExport();
|
||||
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.InPolygon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.json.JsonSlurper
|
||||
import groovy.io.FileType
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.awt.geom.Point2D
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class AddDevShapeData {
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
def devAreaMap = [:]
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def results = new HashSet()
|
||||
def existList = new HashSet()
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDevShapeData();
|
||||
task.run()
|
||||
// task.streetInfo()
|
||||
// task.addStreetToMapping()//通过乡镇街道的点是否都在开发区围栏中,判断是否将此乡镇加入映射表中
|
||||
}
|
||||
|
||||
def addStreetToMapping(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
def devShapes = findDevShapeByCity(fullCityName)
|
||||
if(devShapes){
|
||||
def streetShapes = findStreetShapeByCity(fullCityName)
|
||||
addDevStreetToMapping(fullCityName,streetShapes, devShapes)
|
||||
}
|
||||
}
|
||||
|
||||
results.each {s->
|
||||
println s
|
||||
}
|
||||
existList.each{s->
|
||||
println s
|
||||
}
|
||||
|
||||
println " 新增映射乡镇:"+results.size()+" 已存在乡镇:"+existList.size()
|
||||
|
||||
}
|
||||
|
||||
def addDevStreetToMapping(city,streetShapes, devShapes){
|
||||
def addCount = 0
|
||||
def existsCount = 0
|
||||
|
||||
|
||||
for(def streetObj : streetShapes){
|
||||
|
||||
def coordinates = streetObj.shape.coordinates
|
||||
for(def pointList : coordinates){
|
||||
def objList = findDevInStreetRange(streetObj, pointList, devShapes)
|
||||
if(objList){
|
||||
for(def obj : objList){
|
||||
def existsFlag = isExistStreetByDevName(obj.name, streetObj.street)
|
||||
if(existsFlag){
|
||||
existsCount++
|
||||
existList << "[存在] "+obj.name+" "+ streetObj.street
|
||||
}else{
|
||||
addCount++
|
||||
results << "[新增] "+obj.name+"#"+ streetObj.street
|
||||
if(!streetObj.street){
|
||||
println streetObj
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findDevInStreetRange(streetObj,pointList, devShapes){
|
||||
def devList = []
|
||||
def minDis =200
|
||||
for(def devObj : devShapes){
|
||||
def flagList = []
|
||||
def distances = []
|
||||
for(def pointArr : pointList){
|
||||
// def flag = InsidePolygon.isInArea(pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble(), InsidePolygon.createPoint4(devObj.shape.coordinates))
|
||||
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6,pointArr[0].toString().toDouble()), DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()));
|
||||
def flag = InPolygon.isInArea(point, InPolygon.createPoint(devObj.shape.coordinates))
|
||||
flagList << flag
|
||||
if(!flag){
|
||||
distances << findNearestDistances(pointArr, devObj.shape.coordinates)
|
||||
}
|
||||
}
|
||||
|
||||
def trueCount = statResult(flagList)
|
||||
def pre = trueCount*100/flagList.size()
|
||||
println streetObj.district+" "+streetObj.street +" "+devObj.name +" "+flagList.size()+" "+pre+"%"
|
||||
if(pre>=100){
|
||||
devList << devObj
|
||||
}
|
||||
// else{
|
||||
// if(distances && pre>=50){
|
||||
// def minCount = 0
|
||||
// for(def d : distances){
|
||||
// if(d <= minDis){
|
||||
// minCount++
|
||||
// }
|
||||
// }
|
||||
// def simlarPointPre = minCount*100/distances.size()
|
||||
// if(simlarPointPre>=90){
|
||||
// devList << devObj
|
||||
// }
|
||||
// println "不在多边型范围内的点,距离小于${minDis}米有:"+minCount+" 占比:"+simlarPointPre
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
return devList
|
||||
}
|
||||
|
||||
def findNearestDistances(pointArr, coordinates){
|
||||
def minDis = Integer.MAX_VALUE
|
||||
for(def pointlist : coordinates){
|
||||
for(def devPoint : pointlist){
|
||||
def distance = GeoUtils.calculateDistance(pointArr[0], pointArr[1], devPoint[0], devPoint[1])
|
||||
if(distance < minDis){
|
||||
minDis = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
return minDis
|
||||
}
|
||||
|
||||
def statResult(rows){
|
||||
def count = 0
|
||||
for(def s : rows){
|
||||
if(s){
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
def findStreetShapeByCity(city){
|
||||
def list = []
|
||||
def dbcurosr = administrative_mapping_coll.find(new BasicDBObject("city", city).append("type", "乡镇街道").append("shape", new BasicDBObject('$exists', true)))
|
||||
for(def d : dbcurosr){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
def findDevShapeByCity(city){
|
||||
def list = []
|
||||
def dbcurosr = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("city", city).append('shape', new BasicDBObject('$exists',true)))
|
||||
for(def d : dbcurosr){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def isExistStreetByDevName(devName, street){
|
||||
def list = dev_mapping_CONN.find(new BasicDBObject("name", devName).append("type","乡镇街道").append("township",street)).toList()
|
||||
if(list){
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\'
|
||||
def dir = new File(directoryPath)
|
||||
if (dir.isDirectory()) {
|
||||
dir.eachFileRecurse(FileType.FILES) { file ->
|
||||
process(file.absolutePath)
|
||||
}
|
||||
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
|
||||
} else {
|
||||
println "The specified path is not a directory."
|
||||
}
|
||||
}
|
||||
|
||||
def addDevInfo(file){
|
||||
ExcelUtils.multiThreadingModify(file, {d->
|
||||
// if(!d['I']){
|
||||
def name = d['E']
|
||||
def q = new BasicDBObject()
|
||||
q.put('_id',new ObjectId(d['A']))
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
|
||||
if(obj){
|
||||
if(obj.operator){
|
||||
d['H'] = obj.operator
|
||||
}
|
||||
if(obj.operate_result){
|
||||
d['I'] = obj.operate_result
|
||||
}
|
||||
if(obj.shape){
|
||||
d['T'] = "有"
|
||||
}
|
||||
// if(obj.m_date){
|
||||
// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
|
||||
// }
|
||||
|
||||
// if(devAreaMap){
|
||||
// devAreaMap.each {k,v->
|
||||
// if(obj.name && k && obj.name.contains(k)){
|
||||
// d['M'] = v.area_info
|
||||
// d['N'] = v.count
|
||||
// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }else{
|
||||
// println name+" 空值"
|
||||
// }
|
||||
}
|
||||
// }
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def process(file){
|
||||
def retMap = getNewLocations(file)
|
||||
if(retMap && retMap.gaode.size()>0){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+retMap.name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
|
||||
if(obj){
|
||||
for(def i=0;i<retMap.gaode.size();i++){
|
||||
def udata = new BasicDBObject()
|
||||
def resultMap = [:]
|
||||
def bd_resultMap = [:]
|
||||
|
||||
resultMap.type = "polygon"
|
||||
resultMap.coordinates = [retMap.gaode[i]]
|
||||
|
||||
bd_resultMap.type = "polygon"
|
||||
bd_resultMap.coordinates = [retMap.baidu[i]]
|
||||
if(i>0){
|
||||
obj.removeField("_id")
|
||||
obj.put("create_time", new Date())
|
||||
obj.put("block", retMap.name+"-地块"+i)
|
||||
obj.put("shape", resultMap)
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("bd_shape",bd_resultMap)
|
||||
obj.put("operator", "外包")
|
||||
obj.put( "operate_result" , "已处理")
|
||||
DEVELOPMENT_ZONE_43_COLL.save(obj)
|
||||
}else{
|
||||
udata.put("shape", resultMap)
|
||||
// udata.put("m_date", new Date())
|
||||
udata.put("bd_shape",bd_resultMap)
|
||||
udata.put("operate_result" , "已处理")
|
||||
udata.put("operator", "外包")
|
||||
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getNewLocations(file){
|
||||
def jsonObj = readJson(file)
|
||||
def features = jsonObj.features
|
||||
def gaode_locations = []
|
||||
def baidu_locations = []
|
||||
def name = null
|
||||
def areaMap = [:]
|
||||
if(features && features.size()==1){
|
||||
name = features[0].properties.Name
|
||||
def coordinates = features[0].geometry.coordinates
|
||||
|
||||
def index = 0
|
||||
println file.toString()+" "+name +" 有 "+coordinates.size()+" 块"
|
||||
|
||||
areaMap.count = coordinates.size()
|
||||
def areas = []
|
||||
// if(name.contains("江苏常熟综合保税区")){
|
||||
for(def pointList : coordinates){
|
||||
index ++
|
||||
def gaode_coordinates = []
|
||||
def bd_coordinates = []
|
||||
def points = null
|
||||
if(pointList.size()>1){
|
||||
points = pointList
|
||||
}else{
|
||||
points = pointList[0]
|
||||
}
|
||||
double area = com.fangjia.script.util.GeoHelper.calculateArea(points)
|
||||
println name +" 地块"+index + "围栏面积是: ${area} 平方公里"
|
||||
areas << area
|
||||
def locationstr = ""
|
||||
for(def arrs : points){
|
||||
def gaode_map = GeoUtils.gps84ToGcj02(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
gaode_coordinates << [gaode_map.get("lng"), gaode_map.get("lat")]
|
||||
|
||||
//此高德转百度 有偏差
|
||||
def bd_map = GeoUtils.gcj02ToBd09(gaode_map.get("lat"), gaode_map.get("lng"))
|
||||
// bd_coordinates << [bd_map.get("lng"), bd_map.get("lat")]
|
||||
// 高德转百度 通过接口有次数限制
|
||||
// def baidu = BaiDuApiUtils.geoconv(gaode_map.get("lng"), gaode_map.get("lat"), 3)
|
||||
// bd_coordinates << [baidu.x, baidu.y]
|
||||
|
||||
locationstr = arrs[0].toString()+","+arrs[1].toString()
|
||||
def bdpoint = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()), 6)
|
||||
if(bdpoint){
|
||||
bd_coordinates << bdpoint
|
||||
}
|
||||
}
|
||||
//整个一串一起传超过4029会报错
|
||||
// bd_coordinates = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()-1), 6)
|
||||
gaode_locations << gaode_coordinates
|
||||
if(bd_coordinates){
|
||||
baidu_locations << bd_coordinates
|
||||
}else{
|
||||
println name +" 转换有问题"
|
||||
}
|
||||
|
||||
}
|
||||
areaMap.area_info = areas.join("\n")
|
||||
// }
|
||||
|
||||
}else{
|
||||
println file +" 有多个"
|
||||
}
|
||||
|
||||
devAreaMap.put(name, areaMap)
|
||||
def retMap = [:]
|
||||
retMap.name = name
|
||||
retMap.gaode = gaode_locations
|
||||
retMap.baidu = baidu_locations
|
||||
return retMap
|
||||
}
|
||||
|
||||
def readJson(def file){
|
||||
def jsonSlurper = new JsonSlurper()
|
||||
def jsonFile = new File(file)
|
||||
def jsonText = jsonFile.text
|
||||
def jsonObj = jsonSlurper.parseText(jsonText)
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
def streetInfo(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx", {d->
|
||||
def count = dev_mapping_CONN.count(new BasicDBObject("name", d['E']).append("type", "乡镇街道"))
|
||||
d['L'] = count
|
||||
})
|
||||
}
|
||||
}
|
||||
+448
@@ -0,0 +1,448 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.json.JsonSlurper
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
//补充库中没有详细地址的小区
|
||||
class AddDistrictAddress extends BaseBatch{
|
||||
|
||||
// def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
// def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
//
|
||||
// DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
// public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
//
|
||||
//
|
||||
// public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
//
|
||||
// def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
// DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
//
|
||||
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictAddress(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// addNoAddressDist() //为不精确的地址通过地图补充地址
|
||||
// mapOtherAddress() //有精确地址的寻找其它地址
|
||||
|
||||
// addressSelect()
|
||||
addrUpdateToDB()
|
||||
}
|
||||
|
||||
|
||||
def addNoAddressDist(){
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def index = 0
|
||||
// def allrets = []
|
||||
// for(def c : citys){
|
||||
// index ++
|
||||
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
|
||||
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
|
||||
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
|
||||
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
|
||||
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
|
||||
//
|
||||
// def rets = []
|
||||
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
|
||||
// for(def d : noRoadInfoDists){
|
||||
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
|
||||
// }
|
||||
// allrets.addAll(rets)
|
||||
// if(allrets.size()>50000){
|
||||
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
|
||||
// allrets.clear()
|
||||
// }
|
||||
// }
|
||||
updateDistAddr()
|
||||
}
|
||||
|
||||
|
||||
def mapOtherAddress(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
def allrets = []
|
||||
for(def c : citys){
|
||||
index ++
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address", "addr_alias"])
|
||||
def radInfoDists = haveRoadInfoDistricts(districts)
|
||||
println c +" "+index +" 原始小区量:"+districts.size() +" 有详细地址量:"+radInfoDists.size()
|
||||
radInfoDists = baiduMatch(c, radInfoDists)
|
||||
radInfoDists = gaodeMatch(c, radInfoDists)
|
||||
|
||||
def rets = []
|
||||
rets << ["id", "城市", "区域", "名称", "地址","地址别名", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
|
||||
for(def d : radInfoDists){
|
||||
def addr_alias_str = '';
|
||||
if(d.addr_alias){
|
||||
addr_alias_str = d.addr_alias.join(" ")
|
||||
}
|
||||
rets << [d._id.toString(), d.city, d.region, d.name, d.address, addr_alias_str,d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
|
||||
}
|
||||
allrets.addAll(rets)
|
||||
if(allrets.size()>50000){
|
||||
ExcelUtils.write("D:\\数据\\小区\\小区精确地址补充"+index+".xlsx", allrets, "补充地址")
|
||||
allrets.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addressSelect(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-all-2.xlsx",{d->
|
||||
if(d['G']){
|
||||
d['O'] = AddressUtil.formatAddress(d['B'], d['G'])
|
||||
}
|
||||
if(d['I']){
|
||||
d['P'] = AddressUtil.formatAddress(d['B'], d['I'])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addrUpdateToDB(){
|
||||
def updateDist = 0
|
||||
def addAddrCount = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-入库-2.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def oldAddrAliasCount = 0
|
||||
if(dist.addr_alias){
|
||||
oldAddrAliasCount = dist.addr_alias.size()
|
||||
}
|
||||
def new_addr_alias = null
|
||||
if(d['K'].contains("新增百度地址") || d['K'].contains("双地图新增")){
|
||||
new_addr_alias = getNewAddressAlias(d['L'], dist.address, dist.addr_alias)
|
||||
}
|
||||
if(d['K'].contains("新增高德地址") || d['K'].contains("双地图新增")){
|
||||
new_addr_alias = getNewAddressAlias(d['M'], dist.address, dist.addr_alias)
|
||||
}
|
||||
if(new_addr_alias){
|
||||
updateDist++
|
||||
println updateDist
|
||||
addAddrCount = addAddrCount + new_addr_alias.size()-oldAddrAliasCount
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("addr_alias", new_addr_alias)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
println "修改小区数:"+updateDist
|
||||
println "新增地址数:"+addAddrCount
|
||||
}
|
||||
|
||||
def getNewAddressAlias(new_addr, old_addr, dist_addr_alias){
|
||||
def arrs = new_addr.split(";")
|
||||
def addr_alias = []
|
||||
if(dist_addr_alias){
|
||||
addr_alias = dist_addr_alias
|
||||
}
|
||||
for(def address : arrs){
|
||||
if(old_addr!=address && !addr_alias.contains(address)){
|
||||
addr_alias << address
|
||||
}
|
||||
}
|
||||
return addr_alias
|
||||
}
|
||||
|
||||
def updateDistAddr(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", d['J'])
|
||||
def addr_alias = dist.addr_alias
|
||||
if(!dist.addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
if(d['K'] && !addr_alias.contains(d['K'])){
|
||||
addr_alias << d['K']
|
||||
}
|
||||
|
||||
if(addr_alias){
|
||||
udata.put("addr_alias", addr_alias)
|
||||
}
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
d['L'] = "已更新"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def noDetailRoadInfoDistricts(districts){
|
||||
def rets = []
|
||||
for(def d : districts){
|
||||
if(d.address){
|
||||
def flag = DataHandleUtils.isStandardRoad(d.address)
|
||||
if(!flag){
|
||||
rets << d
|
||||
}
|
||||
}else{
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def haveRoadInfoDistricts(districts){
|
||||
def rets = []
|
||||
for(def d : districts){
|
||||
if(d.address){
|
||||
def flag = DataHandleUtils.isStandardRoad(d.address)
|
||||
if(flag){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = baiduProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.baidu_address = retMap.new_addr
|
||||
d.baidu_old_addr = retMap.old_addr
|
||||
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
def baiduProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
|
||||
for(def name : alias){
|
||||
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
|
||||
if(temp_address){
|
||||
retMap.old_addr = temp_address
|
||||
}
|
||||
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
|
||||
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
|
||||
retMap.new_addr = nineAddrSplit(fullCityName, temp_address)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def nineAddrSplit(city, addr){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
|
||||
def address = ""
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
|
||||
address = jsonObject.get("street")
|
||||
}
|
||||
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
|
||||
address = address + jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
return address
|
||||
}
|
||||
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
|
||||
def dbcursor = BAIDU_POI_COLL.find(q)
|
||||
def address = null
|
||||
for(def d : dbcursor){
|
||||
address = d.addr
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
|
||||
def gaodeMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = gaodeProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.gaode_addr = retMap.new_addr
|
||||
d.gaode_old_addr = retMap.old_addr
|
||||
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
|
||||
def gaodeProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
def address = null
|
||||
for(def name : alias){
|
||||
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
|
||||
if(temp_addr){
|
||||
retMap.old_addr = temp_addr
|
||||
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
|
||||
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
|
||||
retMap.new_addr = nineAddrSplit(fullCityName, temp_addr)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def findGaoDeDatas2(city, region, name){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
|
||||
def body = """
|
||||
{
|
||||
"_source": {
|
||||
"includes": ["cat", "address", "id","name"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"city" : "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"district" : "${region}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"name": {"query": "${name}","operator": "and"}
|
||||
}
|
||||
}
|
||||
// ,{
|
||||
// "match": {
|
||||
// "small_type": "住宅"
|
||||
// }
|
||||
// }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
|
||||
def address = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
def d = hits[0]
|
||||
if(d._source.cat.contains("住宅")){
|
||||
def source = d._source
|
||||
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
|
||||
if(newName == name){
|
||||
address = source.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return address
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+10
-6
@@ -30,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
|
||||
@@ -55,7 +58,7 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
// }
|
||||
|
||||
// checkRangeCount()
|
||||
synch100()
|
||||
// synch100()
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
@@ -378,20 +381,21 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
|
||||
|
||||
def checkRangeCount(){
|
||||
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||
// def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||
|
||||
def dbcursor = administrative_mapping_coll.find(new BasicDBObject("type","ÏçÕò½ÖµÀ")).batchSize(100)
|
||||
def count = 0
|
||||
def index =0
|
||||
for(def d : dbcursor){
|
||||
index ++
|
||||
if(d.location){
|
||||
def coordinates = d.location.coordinates
|
||||
if(d.shape){
|
||||
def coordinates = d.shape.coordinates
|
||||
if(coordinates.size()>1){
|
||||
count++
|
||||
println count +" "+ d.city +" "+d.name
|
||||
println count +" "+ d.city +" "+d.street
|
||||
}
|
||||
}else{
|
||||
println d.city + " "+d.name
|
||||
println d.city + " "+d.street
|
||||
}
|
||||
|
||||
if(index%1000==0){
|
||||
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class AddrAlaisProcess {
|
||||
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new AddrAlaisProcess()
|
||||
// task.run()
|
||||
// task.splitDist()
|
||||
// task.addAddr()
|
||||
task.nineApi()
|
||||
// task.createResult()
|
||||
// task.mergeResult()
|
||||
}
|
||||
|
||||
def splitDist(){
|
||||
def rows = ExcelUtils.read("D:\\数据\\数据建设\\小区\\全量小区地址及地址别名.xlsx",0,1)
|
||||
println "原始总计:"+rows.size()
|
||||
def rets = []
|
||||
for(def d : rows){
|
||||
def addr = d['F']
|
||||
if(d['G']){
|
||||
addr = addr +","+d['G']
|
||||
}
|
||||
|
||||
def addrs = addr.split(",")
|
||||
for(def s : addrs){
|
||||
rets << [d['A'], d['B'], d['C'], d['D'], d['E'], s]
|
||||
}
|
||||
}
|
||||
println "总计:"+rets.size()
|
||||
FileUtils.batchCreateFileByCustomSuffix(rets, "D:\\数据\\数据建设\\小区\\有地址小区地址标准化(一行一个地址).bat", null, "utf-8", false, " ","");
|
||||
}
|
||||
|
||||
def addAddr(){
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\无地址小区.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
// d['F'] = dist.address
|
||||
if(dist.addr_alias){
|
||||
d['G'] = dist.addr_alias.join(",")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def mergeResult(){
|
||||
def rets = ExcelUtils.read("D:\\数据\\数据建设\\小区\\ai-结果-0221.xlsx",0,1)
|
||||
def retMap = [:]
|
||||
for(def r : rets){
|
||||
def num = r['B'].toString().toDouble().toInteger()
|
||||
def list = retMap.get(num)
|
||||
if(!list){
|
||||
list = new HashSet()
|
||||
retMap.put(num, list)
|
||||
}
|
||||
if(r['C']){
|
||||
list << r['C']
|
||||
}
|
||||
|
||||
}
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def num = d['A'].toString().toDouble().toInteger()
|
||||
def aiList = retMap.get(num)
|
||||
if(aiList){
|
||||
d['K'] = aiList.join(",")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def createResult(){
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
|
||||
def results = []
|
||||
// results << ["ID", "城市", "地址"]
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def str = d['C']
|
||||
def addressList = str.split(",")
|
||||
for(def addr : addressList){
|
||||
results << [d['A'].toString().toDouble().toInteger(), d['B'], addr]
|
||||
}
|
||||
})
|
||||
println results.size()
|
||||
FileUtils.batchCreateFileByCustomSuffix(results, "D:\\数据\\数据建设\\小区\\ai-待支行.bat", null, "utf-8", false, " ","");
|
||||
}
|
||||
|
||||
|
||||
def nineApi(){
|
||||
|
||||
def i = 0;
|
||||
while(i<20) {
|
||||
def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
def addrMap = nineAddrSplit(d['D'], d['G'])
|
||||
def addrList = []
|
||||
if(addrMap && addrMap.address){
|
||||
d['H'] = addrMap.road
|
||||
d['I'] = addrMap.road_num
|
||||
}
|
||||
if(d['H']){
|
||||
def data = fullCityMap.get(d['D'])
|
||||
if(data){
|
||||
def roadObj = findBaiduDatas( data.city, d['H'])
|
||||
if(roadObj){
|
||||
d['J'] = "百度存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(d['F']){
|
||||
// def addrs = d['F'].split(",")
|
||||
// def addr_alias = []
|
||||
// for(def addr : addrs){
|
||||
// def map = nineAddrSplit(d['B'], addr)
|
||||
// if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
|
||||
// if(!addrList.contains(map.address)){
|
||||
// addrList << map.address
|
||||
// addr_alias << map.address
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// d['I'] = addr_alias.join(",")
|
||||
// }
|
||||
})
|
||||
println fileName +" 完成 "
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def nineAddrSplit(city, addr){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
|
||||
def address = ""
|
||||
def map = [:]
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
|
||||
map.province = jsonObject.get("province")
|
||||
}else{
|
||||
map.province = ""
|
||||
}
|
||||
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
|
||||
map.city = jsonObject.get("city")
|
||||
}else{
|
||||
map.city = ""
|
||||
}
|
||||
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
|
||||
map.district = jsonObject.get("district")
|
||||
}else{
|
||||
map.district = ""
|
||||
}
|
||||
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
|
||||
address = jsonObject.get("street")
|
||||
map.road = address
|
||||
}
|
||||
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
|
||||
address = address + jsonObject.get("streetNumber")
|
||||
map.road_num = jsonObject.get("streetNumber")
|
||||
}
|
||||
map.address = address
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rows = []
|
||||
citys.each{c->
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","address", "addr_alias"])
|
||||
println c +" "+districts.size()
|
||||
def num = 0
|
||||
for(def d : districts){
|
||||
num++
|
||||
def row = [num, d._id.toString(), d.city, d.region, d.name, d.address]
|
||||
if(d.addr_alias){
|
||||
row << d.addr_alias.join(",")
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区")
|
||||
FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-20250402.bat", null, "utf-8", false, "#","");
|
||||
}
|
||||
|
||||
def findBaiduDatas(city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', Pattern.compile('^道路'))
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
+44
-7
@@ -4,6 +4,7 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -30,7 +31,10 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AutoAddDistrictBatch(), args)
|
||||
// runBatch(new AutoAddDistrictBatch(), args)
|
||||
|
||||
def task = new AutoAddDistrictBatch();
|
||||
task.checkMap()
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -38,7 +42,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
|
||||
for(def c : citys){
|
||||
def noMatchDatas = findNoMatchResult(c)
|
||||
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||
@@ -48,11 +52,10 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
|
||||
fileName = "D:\\数据\\add_dist_info_1.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
|
||||
|
||||
check(fileName)
|
||||
}
|
||||
|
||||
@@ -61,6 +64,11 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
|
||||
if(dists){
|
||||
d['J'] = "库中已存在"
|
||||
}else{
|
||||
dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district")
|
||||
if(dists){
|
||||
d['J'] = "库中别名中存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -154,10 +162,10 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def findGaoDeDatas(city, region, name){
|
||||
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
|
||||
if(d.name == name){ //&& d.shape
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
@@ -250,8 +258,37 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
// q.put('std_tag', '房地产;住宅区')
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
def checkMap(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\写字楼存在性判断-test.xlsx",{d->
|
||||
|
||||
def gaode_obj = findGaodeCount(d['B'], d['C'], d['D'])
|
||||
if(gaode_obj){
|
||||
d['H'] = "是"
|
||||
d['I'] = gaode_obj.type
|
||||
}else{
|
||||
def baidu_obj = findBaiduCount(d['B'], d['C'], d['D'])
|
||||
if(baidu_obj){
|
||||
d['E'] = "是"
|
||||
d['F'] = baidu_obj.std_tag
|
||||
}
|
||||
}
|
||||
|
||||
// def baidu_obj1 = findBaiduCount(d['B'], d['G'], d['H'])
|
||||
// if(baidu_obj1){
|
||||
// d['M'] = "是"
|
||||
// }
|
||||
// def gaode_obj1 = findGaodeCount(d['B'], d['G'], d['H'])
|
||||
// if(gaode_obj1){
|
||||
// d['N'] = "是"
|
||||
// }
|
||||
// def score = CosineSimilarAlgorithm.getSimilarityRatio(d['D'], d['H']);
|
||||
// d['O'] = score
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+563
@@ -0,0 +1,563 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CopyOnWriteArraySet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AutoAddOfficeBatch extends BaseBatch {
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206", 27020)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
// runBatch(new AutoAddOfficeBatch(), args)
|
||||
|
||||
def task = new AutoAddOfficeBatch()
|
||||
// task.repeatOffice()
|
||||
task.repeatDist()
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
|
||||
// def newName = "开放广场".replaceAll(pattern, "")
|
||||
// println newName
|
||||
// findAddOfficeDatas() //查找需要新增的写字楼数据
|
||||
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
|
||||
// addOffice()//新增写字楼
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findAddOfficeDatas() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allAddOffices = []
|
||||
allAddOffices << ["地图ID", "省", "市", "区", "写字楼名称", "地址", "经度", "纬度", "匹配城市", "匹配区域", "匹配名称", "匹配方式"]
|
||||
for (def c : citys) {
|
||||
def cityObj = fullCityMap.get(c)
|
||||
if (cityObj) {
|
||||
def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼")
|
||||
officeMaps = filterOffice(officeMaps)
|
||||
if (officeMaps) {
|
||||
def offices = matchOffice(officeMaps)
|
||||
|
||||
allAddOffices.addAll(offices)
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
|
||||
}
|
||||
|
||||
def checkInBaiduMapOffice() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx", { d ->
|
||||
if (!d['L']) {
|
||||
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
|
||||
if (obj) {
|
||||
d['L'] = "百度存在"
|
||||
d['M'] = obj.std_tag.toString()
|
||||
}
|
||||
}
|
||||
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def filterOffice(officeMaps) {
|
||||
def offices = []
|
||||
def keys = []
|
||||
for (def o : officeMaps) {
|
||||
if (!o) {
|
||||
continue
|
||||
}
|
||||
if (o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")) {
|
||||
println "[ 过滤 ] " + o.city + " " + o.name
|
||||
continue
|
||||
}
|
||||
if (o.name.contains("路写字楼")) {
|
||||
println "[ 过滤 ] " + o.city + " " + o.name
|
||||
continue
|
||||
}
|
||||
|
||||
def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
|
||||
def newName = o.name.replaceAll(pattern, "")
|
||||
o.name = newName
|
||||
def keyName = o.city + "#" + o.district + "#" + o.name
|
||||
if (!keys.contains(keyName)) {
|
||||
offices << o
|
||||
keys << keyName
|
||||
}
|
||||
}
|
||||
println "原:" + officeMaps.size() + " 新:" + offices.size()
|
||||
return offices
|
||||
}
|
||||
|
||||
def matchOffice(offices) {
|
||||
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
def total = offices.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
for (def i = 0; i < offices.size(); i++) {
|
||||
final def nmd = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
addCount.addAndGet(1)
|
||||
|
||||
def office = matchDB(nmd)
|
||||
if (!office) {
|
||||
office = matchApi(nmd)
|
||||
}
|
||||
if (office) {
|
||||
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, office.city, office.region, office.name, office.match_way]
|
||||
println addCount.get() + "/" + total + " [success] " + office.get("region") + " " + office.get("name") + " " + office.match_way
|
||||
} else {
|
||||
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, "", "", "", ""]
|
||||
println addCount.get() + "/" + total + " [fail] " + nmd.city + " " + nmd.name
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def matchDB(nmd) {
|
||||
def simpleCity = RegionData.formatCity(nmd.city)
|
||||
def simpleDistrict = RegionData.formatRegion(simpleCity, nmd.district)
|
||||
if (simpleCity && simpleDistrict) {
|
||||
def office = null
|
||||
def rets = CommonDatautils.findDistrictList(simpleCity, simpleDistrict, nmd.name, "office")
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
office = rets[0]
|
||||
}
|
||||
} else {
|
||||
rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office")
|
||||
if (rets && rets.size() >= 1) {
|
||||
office = rets[0]
|
||||
} else {
|
||||
rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district")
|
||||
if (rets && rets.size() >= 1) {
|
||||
office = rets[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if (office) {
|
||||
office.match_way = "数据库"
|
||||
}
|
||||
return office
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def matchApi(nmd) {
|
||||
def address = nmd.city
|
||||
if (nmd.district) {
|
||||
address = address + nmd.district
|
||||
}
|
||||
address = address + nmd.name
|
||||
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
|
||||
if (!jsonObject) {
|
||||
if (DataUtil.isPreciseAddress(nmd.address)) {
|
||||
address = nmd.city
|
||||
if (nmd.district) {
|
||||
address = address + nmd.district
|
||||
}
|
||||
address = address + nmd.address
|
||||
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
|
||||
}
|
||||
}
|
||||
if (jsonObject) {
|
||||
def udata = [:]
|
||||
udata.put("_id", jsonObject.get("id"))
|
||||
udata.put("city", jsonObject.get("city"))
|
||||
udata.put("region", jsonObject.get("district"))
|
||||
udata.put("name", jsonObject.get("name"))
|
||||
udata.put("match_type", jsonObject.get("propertyType"))
|
||||
udata.put("match_credit", jsonObject.get("credit"))
|
||||
udata.put("match_way", "接口")
|
||||
return udata
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def repeatOffice() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << ["ID", "城市", "区域", "名称", "地址", "可疑ID", "可疑城市", "可疑区域", "可疑名称", "可疑地址", "相似度", "距离"]
|
||||
for (def c : citys) {
|
||||
def offices = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "address", "lat", "lng"])
|
||||
def rets = checkRepeatOfficeBatch(offices)
|
||||
println c + " " + offices.size()
|
||||
allrets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据")
|
||||
}
|
||||
|
||||
def repeatDist(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
|
||||
def index = 0
|
||||
for (def c : citys) {
|
||||
index++
|
||||
if(["宁波","无锡","青岛"].contains(c)){
|
||||
continue
|
||||
}
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
|
||||
def rets = checkRepeatDistrictBatch(c, districts)
|
||||
println c + " " + districts.size()
|
||||
allrets.addAll(rets)
|
||||
println "${index}"+ c +" 运A行完成 "+rets.size()
|
||||
|
||||
if (CommonDatautils.isWindow()) {
|
||||
ExcelUtils.write("D://数据//小区/重复可疑小区-${c}.xlsx", allrets, "数据")
|
||||
}else{
|
||||
ExcelUtils.write("/home/goojia/gj/重复可疑小区-${c}.xlsx", allrets, "数据")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
def checkRepeatDistrictBatch(city,districts) {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def cityId = cityCodeDataMap.get(city)
|
||||
|
||||
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def total = districts.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def sets = new CopyOnWriteArraySet()
|
||||
for (def i = 0; i < total; i++) {
|
||||
final def o1 = districts[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try {
|
||||
addCount.addAndGet(1)
|
||||
// if(!o1.name.contains("四公司")){
|
||||
// continue
|
||||
// }
|
||||
if(!sets.contains(o1._id.toString())){
|
||||
def similarRets = findListSimilar(o1, districts)
|
||||
if (similarRets) {
|
||||
def newPriceMap1 = fjwPriceMap.get(o1._id.toString())
|
||||
def new_price1 = null
|
||||
if(newPriceMap1){
|
||||
new_price1 = newPriceMap1.calc_type
|
||||
}
|
||||
def ailas_str1 = ""
|
||||
if(o1.alias){
|
||||
ailas_str1 = o1.alias.join(",")
|
||||
}
|
||||
def rows = []
|
||||
for(def similar : similarRets){
|
||||
if (!sets.contains(similar._id.toString())) {
|
||||
sets.add(similar._id.toString())
|
||||
def ailas_str = ""
|
||||
if(similar.alias){
|
||||
ailas_str = similar.alias.join(",")
|
||||
}
|
||||
def newPriceMap = fjwPriceMap.get(o1._id.toString())
|
||||
def new_price = null
|
||||
if(newPriceMap){
|
||||
new_price = newPriceMap.calc_type
|
||||
}
|
||||
rows << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
|
||||
// rets << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
|
||||
}
|
||||
}
|
||||
if(rows){
|
||||
sets.add(o1._id.toString())
|
||||
rets << [ o1.city, o1.region, o1.name, ailas_str1,new_price1, o1.address, "主", "主", "主"]
|
||||
rets.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
// latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def checkRepeatOfficeBatch(offices) {
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
def total = offices.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def sets = new CopyOnWriteArraySet()
|
||||
for (def i = 0; i < total; i++) {
|
||||
final def o1 = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
addCount.addAndGet(1)
|
||||
def maxObj = findOneSimilarOffice(o1, offices)
|
||||
if (maxObj) {
|
||||
if (!sets.contains(o1._id.toString())) {
|
||||
sets.add(maxObj._id.toString())
|
||||
rets << [o1._id.toString(), o1.city, o1.region, o1.name, o1.address, maxObj._id.toString(), maxObj.city, maxObj.region, maxObj.name, maxObj.address, maxObj.score, maxObj.distance]
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def findListSimilar(o1, datas){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def ids = []
|
||||
ids << o1._id.toString()
|
||||
for (def o2 : datas) {
|
||||
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region) {
|
||||
|
||||
// if(!o2.name.contains("四公司")){
|
||||
// continue
|
||||
// }
|
||||
|
||||
def name1 = o1.name//.replace("小区","").replace("别墅","")
|
||||
def name2 = o2.name//.replace("小区","").replace("别墅","")
|
||||
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
|
||||
def addrScore = null
|
||||
if(o1.address && o2.address){
|
||||
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
|
||||
}
|
||||
def str = null
|
||||
if(o1.name.length() > o2.name.length()){
|
||||
str = o1.name.replace(o2.name, "")
|
||||
}else{
|
||||
str = o2.name.replace(o1.name, "")
|
||||
}
|
||||
if (nameScore > 0.6 && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
|
||||
o2.nameScore = nameScore
|
||||
o2.addrScore = addrScore
|
||||
o2.score = nameScore
|
||||
rets << o2
|
||||
ids << o2._id.toString()
|
||||
def maxObj = findMaxSimilar(ids, o2, datas)
|
||||
while(maxObj){
|
||||
maxObj.nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(maxObj.name, o1.name);
|
||||
if(o1.address && maxObj.address){
|
||||
maxObj.addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, maxObj.address);
|
||||
}
|
||||
maxObj.score = maxObj.nameScore
|
||||
rets << maxObj
|
||||
ids << maxObj._id.toString()
|
||||
maxObj = findMaxSimilar(ids, maxObj, datas)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rets.sort{a,b->
|
||||
def a1 = a.score
|
||||
def b1 = b.score
|
||||
return b1.compareTo(a1)
|
||||
}
|
||||
|
||||
if (rets.size()>2) {
|
||||
return rets[0..2]
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def findMaxSimilar(ids, o1, datas){
|
||||
def maxObj = null
|
||||
def maxScore = -1000
|
||||
for (def o2 : datas) {
|
||||
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region && !ids.contains(o2._id.toString())) {
|
||||
def name1 = o1.name//.replace("小区","").replace("别墅","")
|
||||
def name2 = o2.name//.replace("小区","").replace("别墅","")
|
||||
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
|
||||
def addrScore = null
|
||||
if(o1.address && o2.address){
|
||||
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
|
||||
}
|
||||
def str = null
|
||||
if(o1.name.length() > o2.name.length()){
|
||||
str = o1.name.replace(o2.name, "")
|
||||
}else{
|
||||
str = o2.name.replace(o1.name, "")
|
||||
}
|
||||
if (nameScore > 0.6 && nameScore>maxScore && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
|
||||
maxObj = o2
|
||||
o2.nameScore = nameScore
|
||||
o2.addrScore = addrScore
|
||||
o2.score = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return maxObj
|
||||
}
|
||||
|
||||
def findOneSimilarOffice(o1, offices) {
|
||||
def maxScore = 0
|
||||
def maxObj = null
|
||||
for (def o2 : offices) {
|
||||
if (o1._id.toString() != o2._id.toString()) {
|
||||
double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name);
|
||||
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address, o2.address);
|
||||
if (nameScore + addrScore > maxScore) {
|
||||
maxScore = nameScore + addrScore
|
||||
maxObj = o2
|
||||
}
|
||||
}
|
||||
}
|
||||
def distance = -1000
|
||||
if (o1.lat && maxObj && maxObj.lat) {
|
||||
distance = GeoHelper.GetDistance(o1.lat / 10000000000.0, o1.lng / 10000000000.0, maxObj.lat / 10000000000.0, maxObj.lng / 10000000000.0)
|
||||
}
|
||||
if (maxObj) {
|
||||
maxObj.distance = distance
|
||||
maxObj.score = maxScore
|
||||
return maxObj
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
// q.put('std_tag', '房地产;住宅区')
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
def addOffice(){
|
||||
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
|
||||
if(d['L']!='人工删除'){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(city && region){
|
||||
def locationstr = d['G'].toString()+","+d['H'].toString()
|
||||
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
|
||||
def udata = new BasicDBObject()
|
||||
if(bdpoint){
|
||||
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
|
||||
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
|
||||
}
|
||||
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
udata.put("name", d['E'])
|
||||
udata.put("address", d['I'])
|
||||
udata.put("cat", "office")
|
||||
udata.put("status", 0)
|
||||
udata.put("c_date",new Date())
|
||||
udata.put("m_date",new Date())
|
||||
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
|
||||
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
|
||||
|
||||
def estate_type = null
|
||||
if(d['M'] && d['M'].contains("园区")){
|
||||
estate_type = "创意园区"
|
||||
}
|
||||
|
||||
if(!estate_type && d['E'].contains("创意")){
|
||||
estate_type = "创意园区"
|
||||
}
|
||||
if(estate_type){
|
||||
udata.put("estate_type2", estate_type)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map", markMap)
|
||||
DbUtils.seaweed.save(udata)
|
||||
println d['E'] +" "+udata._id.toString()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findNewFjwAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance235.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
def findNewFjwAvgPriceList(def cityid, date){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr,gujia_bds_id from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujiabds_clc_yrmo_val=${date}"
|
||||
def retMap = [:]
|
||||
pgsqlInstance235.eachRow(sql, { r ->
|
||||
def map = [:]
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
retMap.put(r.gujia_bds_id, map)
|
||||
})
|
||||
return retMap
|
||||
}
|
||||
|
||||
}
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class BusinessQuarterProcess {
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020)
|
||||
|
||||
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new BusinessQuarterProcess()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
// synchBusinessData()
|
||||
// updateBusinessQuarter()
|
||||
// addRegion()
|
||||
synchBusinessToEs()
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
def list = CommonDatautils.queryRangeByCat("区域")
|
||||
println list.size()
|
||||
def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2")))
|
||||
for(def d : dbcursor){
|
||||
if(d.district){
|
||||
continue
|
||||
}
|
||||
for(def regionShape : list){
|
||||
if(d.short_city!=regionShape.city_short){
|
||||
continue
|
||||
}
|
||||
def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates))
|
||||
if(flag){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district", regionShape.mzb_district_name)
|
||||
udata.put("short_district", regionShape.district_short)
|
||||
bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def synchBusinessData(){
|
||||
def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList()
|
||||
synchMongoProcess(datas)
|
||||
|
||||
}
|
||||
|
||||
def synchMongoProcess(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
createData(d)
|
||||
println d.city +" "+totalCount.get() +"/"+datas.size()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def createData(d){
|
||||
def row = new BasicDBObject()
|
||||
row.put("_id", d._id)
|
||||
row.put("short_province", d.province)
|
||||
def short_city = RegionData.formatCity(d.city)
|
||||
row.put("short_city", short_city)
|
||||
row.put("name", d.name)
|
||||
row.put("bd_location", ["lon":DataHandleUtils.getDoubleRound(6,d.longitude), "lat":DataHandleUtils.getDoubleRound(6,d.latitude)])
|
||||
|
||||
def data = fullCityMap.get(short_city)
|
||||
if(data){
|
||||
row.put("province", data.province)
|
||||
row.put("city", data.city)
|
||||
}else{
|
||||
println d.city + " 没找到简称"
|
||||
}
|
||||
|
||||
|
||||
def gdPoints = FangjiaClientUtils.convert(d.longitude+","+d.latitude, 3)
|
||||
if(gdPoints){
|
||||
row.put("location", ["lon":DataHandleUtils.getDoubleRound(6,gdPoints[0]), "lat":DataHandleUtils.getDoubleRound(6,gdPoints[1])])
|
||||
}
|
||||
def border = d.border
|
||||
def bd_shapes = []
|
||||
def gd_shapes = []
|
||||
if(border){
|
||||
def arrs = border.split(";")
|
||||
if(border.contains("|")){
|
||||
println d._id.toString()+" 可能含多个围栏"
|
||||
}
|
||||
|
||||
for(def i=0;i<arrs.size();i++){
|
||||
def str = arrs[i]
|
||||
def arr = str.split(",")
|
||||
def bd_lng = DataHandleUtils.getDoubleRound(6, arr[0].toString().toDouble())
|
||||
def bd_lat = DataHandleUtils.getDoubleRound(6, arr[1].toString().toDouble())
|
||||
if(!bd_shapes.contains([bd_lng, bd_lat]) ){
|
||||
bd_shapes << [bd_lng, bd_lat]
|
||||
}
|
||||
if(bd_shapes.contains([bd_lng, bd_lat]) && i+1==arrs.size() && bd_shapes[0]==[bd_lng, bd_lat]){
|
||||
bd_shapes << [bd_lng, bd_lat]
|
||||
}
|
||||
def tempGdPoints = FangjiaClientUtils.convert(bd_lng+","+bd_lat, 3)
|
||||
if(tempGdPoints){
|
||||
def gdlist = [DataHandleUtils.getDoubleRound(6,tempGdPoints[0]), DataHandleUtils.getDoubleRound(6,tempGdPoints[1])]
|
||||
if(!gd_shapes.contains(gdlist)) {
|
||||
gd_shapes << gdlist
|
||||
}
|
||||
if(gd_shapes.contains(gdlist) && i+1==arrs.size() && gd_shapes[0]==gdlist){
|
||||
gd_shapes << gdlist
|
||||
}
|
||||
}
|
||||
}
|
||||
if(bd_shapes[0] != bd_shapes[bd_shapes.size()-1]){
|
||||
bd_shapes << bd_shapes[0]
|
||||
gd_shapes << gd_shapes[0]
|
||||
}
|
||||
}
|
||||
def gdMap = [:]
|
||||
gdMap.put("type" , "polygon")
|
||||
gdMap.put("coordinates", [gd_shapes])
|
||||
row.put("shape", gdMap)
|
||||
|
||||
def bdMap = [:]
|
||||
bdMap.put("type" , "polygon")
|
||||
bdMap.put("coordinates", [bd_shapes])
|
||||
row.put("bd_shape", bdMap)
|
||||
row.put("c_date", new Date())
|
||||
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
|
||||
}
|
||||
def updateBusinessQuarter(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
if(["上海","北京"].contains(c)){
|
||||
continue
|
||||
}
|
||||
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
|
||||
businessProcess(c, districts, cityBusinessList)
|
||||
}
|
||||
}
|
||||
|
||||
def businessProcess(c,districts, cityBusinessList){
|
||||
def updateIndex = 0
|
||||
for(def d : districts){
|
||||
if(!d.business_quarter){
|
||||
def businessQuarter = findBusinessByRange(d, cityBusinessList)
|
||||
if(businessQuarter){
|
||||
updateIndex ++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("business_quarter", businessQuarter)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println c +" "+ districts.size() +" "+updateIndex
|
||||
}
|
||||
def findBusinessByRange(d, cityBusinessList){
|
||||
def name= null
|
||||
for(def shapeObj : cityBusinessList){
|
||||
if(d.lng){
|
||||
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(shapeObj.shape.coordinates))
|
||||
if(flag){
|
||||
name = shapeObj.name
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
def synchBusinessToEs(){
|
||||
def dbcursor = bk_business_coll.find(new BasicDBObject())
|
||||
def rows = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
if(index<5000){
|
||||
continue
|
||||
}
|
||||
// rows << d
|
||||
// if(rows.size()>=100){
|
||||
// updateByEs(rows)
|
||||
// rows.clear()
|
||||
// }
|
||||
updateByEs(d, index)
|
||||
}
|
||||
}
|
||||
|
||||
def updateByEs(row, index){
|
||||
BulkRequest request = new BulkRequest();
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
int processedCount = 0;
|
||||
|
||||
def id = row.get("_id").toString()
|
||||
def doc = [:]
|
||||
doc.put("province",row.province)
|
||||
doc.put("short_province",row.short_province)
|
||||
doc.put("city",row.city)
|
||||
doc.put("short_city",row.short_city)
|
||||
doc.put("district",row.district)
|
||||
doc.put("short_district",row.short_district)
|
||||
doc.put("name", row.name)
|
||||
doc.put("location", row.location)
|
||||
doc.put("bd_location", row.bd_location)
|
||||
doc.put("shape", row.shape)
|
||||
doc.put("bd_shape", row.bd_shape)
|
||||
|
||||
|
||||
request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON));
|
||||
if (request.numberOfActions() >= 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
// println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+54
-19
@@ -5,21 +5,15 @@ import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.joda.time.Instant
|
||||
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DevProcess {
|
||||
@@ -64,6 +58,7 @@ class DevProcess {
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
task.synchDevTo100()
|
||||
// task.statShape()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
@@ -77,7 +72,7 @@ class DevProcess {
|
||||
// println DateUtils.add("2024-7-24", 7);
|
||||
|
||||
// task.addKfc()
|
||||
// task.devToPark() //开发区下辖园区
|
||||
// task.devToPark() //开发区下辖园
|
||||
// task.checkTown()
|
||||
|
||||
// task.repeatSave()
|
||||
@@ -87,6 +82,9 @@ class DevProcess {
|
||||
// task.matchDevShape()//匹配开发区围栏
|
||||
|
||||
// task.addQiDevResult() //获取齐梁数据结果
|
||||
|
||||
// task.updateDevShape()//修改开发区结构
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -442,30 +440,33 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
|
||||
if(!d['B'] && d['A']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
|
||||
if(!d['E'] && d['D']){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d['A'])
|
||||
// q.put("city",d['G'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['A']]))
|
||||
def name = d['A'].replace('经济技术开发区','经济开发区')
|
||||
// q.put("name", d['D'])
|
||||
q.put("city",d['C'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['D']]))
|
||||
def name = d['D'] //.replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['I'] = "库中有开发区-相似2"
|
||||
d['G'] = "相似"
|
||||
// d['K'] = obj.name
|
||||
// d['C'] = obj._id.toString()
|
||||
d['B'] = obj.name
|
||||
// if(obj.name == d['A']){
|
||||
// d['E'] = "相同"
|
||||
d['E'] = obj.name
|
||||
// if(obj.name == d['D']){
|
||||
// d['G'] = "相同"
|
||||
// }else{
|
||||
// d['E'] = "包含"
|
||||
// d['G'] = "包含"
|
||||
// }
|
||||
if(obj.shape){
|
||||
d['F'] = "是"
|
||||
}
|
||||
// d['E'] = "别名"
|
||||
// d['F'] = obj.name
|
||||
}
|
||||
}
|
||||
},1);
|
||||
},0);
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
@@ -794,6 +795,18 @@ class DevProcess {
|
||||
development_zone_CONN.insert(rets)
|
||||
}
|
||||
|
||||
def statShape(){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("shape", new BasicDBObject('$exists', true)))
|
||||
def index = 0
|
||||
for(def r : dbcursor){
|
||||
index ++
|
||||
def coordinates = r.shape.coordinates
|
||||
println index+" "+r.name+" "+coordinates.size()
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
|
||||
@@ -1148,4 +1161,26 @@ class DevProcess {
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateDevShape(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("shape", new BasicDBObject('$exists', true))
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
if(d.name!="福州长乐国际机场综合保税区"){
|
||||
def udata = new BasicDBObject()
|
||||
if(d.shape.coordinates){
|
||||
d.shape.coordinates = [d.shape.coordinates]
|
||||
udata.put("shape", d.shape)
|
||||
}
|
||||
if(d.bd_shape && d.bd_shape.coordinates){
|
||||
d.bd_shape.coordinates = [d.bd_shape.coordinates]
|
||||
udata.put("bd_shape", d.bd_shape)
|
||||
}
|
||||
def updateQuery = new BasicDBObject("_id",d._id)
|
||||
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.script.gj.ProxyHttpUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
class DevQlCheck {
|
||||
|
||||
//http://146.56.217.44:8088/uams/index
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevQlCheck()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def devName = "福建龙海经济开发区"
|
||||
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",devName))
|
||||
def pois = readCSV("C:\\Users\\gaojuan\\Downloads\\漳州市POI.csv")
|
||||
def devPois = []
|
||||
devPois << ["省", "城市", "区域", "名称", "地址", "类型", "经度", "纬度", "齐梁开发区"]
|
||||
for(def poi : pois){
|
||||
def flag = InsidePolygon.isInArea(poi.lng.toString().toDouble(), poi.lat.toString().toDouble(), InsidePolygon.createPoint4(obj.shape.coordinates))
|
||||
if(flag){
|
||||
devPois << [poi.province, poi.city, poi.district, poi.name, poi.address, poi.cat, poi.lng, poi.lat]
|
||||
}
|
||||
}
|
||||
println "共:"+devPois.size()+" 个"
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx", devPois, "齐梁开发区")
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx",{d->
|
||||
def address = d['B']+d['C']+d['D']
|
||||
// if(d['E']){
|
||||
// address = address + d['E']
|
||||
// }
|
||||
Thread.sleep(3000)
|
||||
def s = getQlResult(address)
|
||||
d['I'] = s
|
||||
index ++
|
||||
println index+" "+ s
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
|
||||
def getQlResult(address){
|
||||
def params = [
|
||||
'address': address,
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
try{
|
||||
def str = ProxyHttpUtils.getProxyResult(targetUrl, params);
|
||||
println address +" "+ str
|
||||
JSONObject result = JSONObject.fromObject(str)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println address +" "+e.getMessage()
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("province", arrs[0])
|
||||
o.put("city", arrs[1])
|
||||
o.put("district", arrs[2])
|
||||
o.put("name", arrs[3])
|
||||
o.put("township", arrs[4])
|
||||
o.put("address", arrs[5])
|
||||
o.put("cat", arrs[6])
|
||||
o.put("lng", arrs[7])
|
||||
o.put("lat", arrs[8])
|
||||
o.put("data_source", arrs[9])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println filename+" 文件异常"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+3
-4
@@ -7,7 +7,6 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -34,8 +33,8 @@ class DevUpdateTestBatch {
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
task.addDevAddress(path)
|
||||
task.addDev(path)
|
||||
// task.addDevAddress(path)
|
||||
//
|
||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||
|
||||
@@ -168,7 +167,7 @@ class DevUpdateTestBatch {
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},2)
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+140
-39
@@ -1,12 +1,14 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -28,14 +30,24 @@ class GXDdistrictMatch {
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
task.addOldMatch2()
|
||||
// task.addOldMatch2()
|
||||
// task.runDb()
|
||||
// task.hainanMatch()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
// task.runMatch()
|
||||
|
||||
// task.checkDist()
|
||||
|
||||
task.addPrice()
|
||||
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
@@ -94,6 +106,46 @@ class GXDdistrictMatch {
|
||||
|
||||
}
|
||||
|
||||
def hainanMatch(){
|
||||
def fileName = "D:\\数据\\小区\\海南未匹配地址-1.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['D'] && d['F'] ){//
|
||||
// def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['I'])
|
||||
// if(jsonObject){
|
||||
//// if(jsonObject.get("district") && jsonObject.get("district")!='null'){
|
||||
//// d['AO'] = jsonObject.get("district")
|
||||
//// }
|
||||
// if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
// d['AP'] = jsonObject.get("province")
|
||||
// }
|
||||
// if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
// d['AI'] = jsonObject.get("city")
|
||||
// }
|
||||
// }
|
||||
def formatCity = RegionData.formatCity(d['F'])
|
||||
def fullCityObj = fullCityMap.get(formatCity)
|
||||
if(fullCityObj){
|
||||
// def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
|
||||
|
||||
def obj = findDatasByCityAndName(formatCity, d['D'].replaceAll('[·\\(\\)()]|·', ''))
|
||||
if(obj){
|
||||
d['U'] = "楼盘抓取库"
|
||||
d['V'] = obj.city
|
||||
d['W'] = obj.region
|
||||
d['X'] = obj.name
|
||||
d['Y'] = obj.property_type
|
||||
d['Z'] = obj.address
|
||||
def loc = obj.location
|
||||
if(loc){
|
||||
d['AA'] = loc.x
|
||||
d['AB'] = loc.y
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def runDb(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
@@ -238,43 +290,41 @@ class GXDdistrictMatch {
|
||||
|
||||
def saveToDB(){
|
||||
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
|
||||
def fileName = "D:\\数据\\小区\\海南新增小区.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['A']){
|
||||
if(d['A'] && d['C']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def alias = []
|
||||
if(d['G'] && name!=d['G']){
|
||||
alias << d['G']
|
||||
}
|
||||
// def alias = []
|
||||
// if(d['G'] && name!=d['G']){
|
||||
// alias << d['G']
|
||||
// }
|
||||
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
// alias << d['AI']
|
||||
// }
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
// if(alias){
|
||||
// udata.put("alias", alias)
|
||||
// }
|
||||
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['E'])
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("status", 0)
|
||||
udata.put("name_cat", 0)
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
// udata.put("username", "高娟")
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
// if(d['AH']){
|
||||
// type = d['AH']
|
||||
// }else{
|
||||
// type = d['Y']
|
||||
// }
|
||||
if(d['D']){
|
||||
type = d['D']
|
||||
}
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
@@ -282,26 +332,50 @@ class GXDdistrictMatch {
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
udata.put("estate_type2", estateType2)
|
||||
|
||||
if(d['F']){
|
||||
Long lng = d['F']*10000000000
|
||||
udata.put("lng", lng)
|
||||
}
|
||||
if(d['E']){
|
||||
Long lat = d['E'] * 10000000000
|
||||
Long lat = d['G'] * 10000000000
|
||||
udata.put("lat", lat)
|
||||
}
|
||||
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
def opt = "add"
|
||||
if(s && s=='成功'){
|
||||
saveOptLog(udata.city, udata.region, udata.name, opt)
|
||||
udata.put("lng", lng)
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(d['F'].toString().toDouble(), d['G'].toString().toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
if(lng2){
|
||||
Long lng2_new = lng2 *10000000000L
|
||||
Long lat2_new = lat2 *10000000000L
|
||||
udata.put("lng2", lng2_new)
|
||||
udata.put("lat2", lat2_new)
|
||||
}
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map",markMap)
|
||||
// udata.put("is_new", 0)
|
||||
DbUtils.seaweed.save(udata)
|
||||
saveOptLog(udata.city, udata.region, udata.name, "add")
|
||||
// def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println name
|
||||
// def opt = "add"
|
||||
// if(s && s=='成功'){
|
||||
// saveOptLog(udata.city, udata.region, udata.name, opt)
|
||||
// }
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def checkDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\海南未匹配地址-1.xlsx", { d ->
|
||||
if(d['V'] && d['W'] && d['X']){
|
||||
matching(d['V'], d['W'], d, d['X'], null, null)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(city, region, d){
|
||||
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
|
||||
@@ -391,7 +465,7 @@ class GXDdistrictMatch {
|
||||
if(rets && rets.size()>=1){
|
||||
d = setRowValue(rets[0], d, "名称在地址中"+str)
|
||||
}else{
|
||||
if (d['D']) {
|
||||
if (d['D'] && alias) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
@@ -449,14 +523,14 @@ class GXDdistrictMatch {
|
||||
|
||||
def setRowValue(dbo, d, remark){
|
||||
if (dbo) {
|
||||
d['V'] = dbo.city
|
||||
d['W'] = dbo.region
|
||||
d['X'] = dbo.name
|
||||
d['Y'] = dbo.estate_type2
|
||||
d['Z'] = dbo.status
|
||||
d['AA'] = dbo.name_cat
|
||||
d['U'] = remark
|
||||
d['AH'] = dbo.address
|
||||
// d['V'] = dbo.city
|
||||
// d['W'] = dbo.region
|
||||
// d['X'] = dbo.name
|
||||
// d['Y'] = dbo.estate_type2
|
||||
d['AK'] = dbo.status
|
||||
// d['AA'] = dbo.name_cat
|
||||
d['AJ'] = remark
|
||||
// d['AH'] = dbo.address
|
||||
}
|
||||
return d;
|
||||
}
|
||||
@@ -510,6 +584,33 @@ class GXDdistrictMatch {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def findDatasByCityAndName(def city, def name){
|
||||
def sql = "select * from community where city='${city}' and name='${name}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
println d.property_type
|
||||
})
|
||||
|
||||
if(rets){
|
||||
def obj = null
|
||||
for(def r : rets){
|
||||
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
|
||||
obj = r
|
||||
break
|
||||
}
|
||||
}
|
||||
if(obj){
|
||||
return obj;
|
||||
}else{
|
||||
return rets[0];
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, type){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
|
||||
+4
-5
@@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* Ãû¼¼¯
|
||||
*/
|
||||
@@ -31,12 +29,13 @@ class MinglujiData {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def prov = "河北省"
|
||||
def city = "石家庄市"
|
||||
def prov = "½ËÕÊ¡"
|
||||
def city = "ÎÞÎý"
|
||||
exportDatas(city)
|
||||
// exportDatas(city)
|
||||
// matchDev(prov, city)
|
||||
// matchCommunity(city)
|
||||
fiveLevelSplit(city)
|
||||
// fiveLevelSplit(city)
|
||||
// nineLevelSplit(city)
|
||||
// tempMatchDev(prov)
|
||||
|
||||
|
||||
+185
-3
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -39,12 +46,13 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
// run()
|
||||
oldPoiMagre()
|
||||
}
|
||||
|
||||
def run(){
|
||||
if (CommonDatautils.isWindow()) {
|
||||
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
|
||||
TAGS = getAllPoiType("D:\\数据\\数据建设\\Poi\\POI分类白名单.xlsx")
|
||||
}else{
|
||||
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
|
||||
}
|
||||
@@ -145,6 +153,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
saveObj.data_id = map.map_id
|
||||
saveObj.data_source = "¸ßµÂµØÍ¼"
|
||||
saveObj.data_source_year = "2024-11"
|
||||
saveObj.parent_id = map.parent_id
|
||||
|
||||
if(saveObj.cat && TAGS.contains(saveObj.cat)){
|
||||
saveObj.status = 0
|
||||
@@ -314,7 +323,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
rows.each{map->
|
||||
// def map = rows.get(key)
|
||||
|
||||
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
|
||||
|
||||
// ÅúÁ¿Ìá½»Êý¾Ý
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
@@ -346,4 +355,177 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def oldPoiMagre(){
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
def index = 0
|
||||
for(def cityObj : cityList){
|
||||
index ++
|
||||
if(index <291){
|
||||
continue
|
||||
}
|
||||
def fileName = "/home/goojia/gj/poi/old_poi/merge_poi/${cityObj.city}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
|
||||
}
|
||||
println fileName
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
def results = readCSV(fileName)
|
||||
println "运行城市:"+cityObj.province+" " +cityObj.city+" "+results.size()
|
||||
if(results){
|
||||
def megerMap = poiRepeat(cityObj.city, results)
|
||||
println "待保存:" + cityObj.city +" "+megerMap.size()
|
||||
save(megerMap.values())
|
||||
}
|
||||
}else{
|
||||
println fileName +" 不存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def poiRepeat(city, oldPois){
|
||||
def rets = new ConcurrentHashMap()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(oldPois.size())
|
||||
def townMap = CommonDatautils.queryTownByCity(city)
|
||||
def esMap = findEsPoiByCity(city)
|
||||
|
||||
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
for(def i=0;i<oldPois.size();i++){
|
||||
final def p = oldPois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def nameMap = esMap.get(p.district)
|
||||
def nameKeys = nameMap.keySet()
|
||||
def key = p.name.replace("-","").replace("·","")
|
||||
if(!nameKeys.contains(key)){
|
||||
def simlarObj = null
|
||||
if(esMap){
|
||||
simlarObj = findSimlarPoi(p, esMap)
|
||||
}
|
||||
if(!simlarObj){
|
||||
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
|
||||
if(!rets.containsKey(newKey)){
|
||||
p.data_source_year = "2023-08"
|
||||
rets.put(newKey, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findSimlarPoi(ret, esMap){
|
||||
def simlarObj = null
|
||||
def region = ""
|
||||
if(ret.district){
|
||||
region = ret.district
|
||||
}
|
||||
def regionMap = esMap.get(region)
|
||||
if(regionMap){
|
||||
def keys = regionMap.keySet()
|
||||
|
||||
for(def key : keys){
|
||||
def esValue = regionMap.get(key)
|
||||
if(esValue.township && ret.township){
|
||||
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
||||
if(ret.location && ret.location.lon){
|
||||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||
if(distance <100){
|
||||
simlarObj = esValue
|
||||
println esValue.name +" "+ret.name+" "+distance
|
||||
break;
|
||||
}
|
||||
}else{
|
||||
println ret.data_id +" 无经纬度信息"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "无数据"
|
||||
}
|
||||
|
||||
return simlarObj
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def map = new HashMap()
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "map_poi_2024", queryBuilder, { String id, Map d->
|
||||
def district = "嘉峪关市"
|
||||
if(d.district){
|
||||
district = d.district
|
||||
}
|
||||
def regionMap = map.get(district)
|
||||
if(!regionMap){
|
||||
regionMap = new HashMap()
|
||||
}
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
regionMap.put(d.name.replace("-","").replace("·",""), o)
|
||||
// map.put(district, regionMap)
|
||||
map.put(district, regionMap)
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("province",arrs[0])
|
||||
o.put("city" , arrs[1])
|
||||
o.put("district",arrs[2])
|
||||
o.put("name",arrs[3])
|
||||
o.put("township",arrs[4])
|
||||
o.put("address",arrs[5])
|
||||
o.put("cat",arrs[6])
|
||||
o.put("type",arrs[7])
|
||||
o.put("secondary_type",arrs[8])
|
||||
o.put("small_type" ,arrs[9])
|
||||
if(arrs[10]){
|
||||
o.put("data_id",arrs[10])
|
||||
}else{
|
||||
o.put("data_id",arrs[11])
|
||||
}
|
||||
|
||||
o.put("spell_name",arrs[12])
|
||||
def location = [:]
|
||||
if(arrs[13]){
|
||||
location.lon = arrs[13].toString().toDouble()
|
||||
location.lat = arrs[14].toString().toDouble()
|
||||
}
|
||||
o.put("location",location)
|
||||
o.put("data_source",arrs[15])
|
||||
o.put("status",arrs[16])
|
||||
o.put("version",arrs[17])
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class NewPoiData {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new NewPoiData()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
// export()
|
||||
// exportNetWorkOffice()
|
||||
// matchByCity()
|
||||
// noMatchData()
|
||||
calcuteDistance()
|
||||
}
|
||||
|
||||
def noMatchData(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\未匹配写字楼2.xlsx", { d ->
|
||||
if(d['H']!='地图'){
|
||||
def map = CommonDatautils.findOneThirdNetWorkOffice(d['A'], d['B'], d['C'])
|
||||
if(map){
|
||||
d['E'] = map.lng
|
||||
d['F'] = map.lat
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
def matchByCity(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\入库写字楼.xlsx",{d->
|
||||
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['A']+d['B']+d['C'], "office", false)
|
||||
def formatAddress = null
|
||||
if(d['D']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
d['G'] = formatAddress
|
||||
}
|
||||
if(!jsonObject && formatAddress){
|
||||
if(DataUtil.isPreciseAddress(formatAddress)){
|
||||
def address = d['A']
|
||||
if(d['B']){
|
||||
address = address + d['B']
|
||||
}
|
||||
address = address + formatAddress
|
||||
jsonObject = FangjiaClientUtils.newMatch(d['A'], address, "office", false)
|
||||
}
|
||||
}
|
||||
if(jsonObject) {
|
||||
d['H'] = jsonObject.get("id")
|
||||
d['I'] = jsonObject.get("city")
|
||||
d['J'] = jsonObject.get("district")
|
||||
d['K'] = jsonObject.get("name")
|
||||
d['L'] = jsonObject.get("propertyType")
|
||||
d['M'] = jsonObject.get("credit")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def export(){
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
|
||||
rets.addAll(CommonDatautils.findNewPoiDatas())
|
||||
ExcelUtils.write("d:\\数据\\poi写字楼.xlsx", rets, "写字楼")
|
||||
}
|
||||
|
||||
def exportNetWorkOffice(){
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
|
||||
rets.addAll(CommonDatautils.findThirdNetWorkOffice())
|
||||
ExcelUtils.write("d:\\数据\\poi写字楼-网站.xlsx", rets, "写字楼")
|
||||
}
|
||||
|
||||
def calcuteDistance(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\写字楼测试-1118-新增.xlsx", { d ->
|
||||
if(d['F']){
|
||||
if(d['A'].contains("北京")){
|
||||
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 39.996743, 116.481075)
|
||||
d['K'] = distance
|
||||
}else if(d['A'].contains("烟台")){
|
||||
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 37.459645, 121.438616)
|
||||
d['K'] = distance
|
||||
}
|
||||
}
|
||||
}, 1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\入库写字楼.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def name = d['C']
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['D']
|
||||
if(address){
|
||||
if(d['G']){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
}
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def key = city+"##"+region+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","office")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+9
-11
@@ -1,22 +1,20 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class PoiNineSplitBatch extends BaseBatch{
|
||||
class PoiNineSplitBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new PoiNineSplitBatch(), args)
|
||||
public static void main(args) {
|
||||
def task = new PoiNineSplitBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
// @Override
|
||||
// void process(Object args) {
|
||||
// run()
|
||||
// }
|
||||
def run(){
|
||||
def i = 0;
|
||||
while(i<360) {
|
||||
|
||||
+58
-7
@@ -4,9 +4,12 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
@@ -31,12 +34,16 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
seaweedRegionStandard()
|
||||
// poiRegionStandard()
|
||||
// seaweedRegionStandard()
|
||||
poiRegionStandard()
|
||||
// poiRegionStandard()
|
||||
//通过乡镇街道修改区域 如:东莞市
|
||||
// updateRegionByTown()
|
||||
}
|
||||
|
||||
def seaweedRegionStandard(){
|
||||
seaweedRegionStandardByCat("district")
|
||||
seaweedRegionStandardByCat("office")
|
||||
}
|
||||
def poiRegionStandard(){
|
||||
|
||||
@@ -93,7 +100,7 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
|
||||
def seaweedRegionStandardByCat(cat){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["伊春"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def rMap = regionMap.get(city)
|
||||
@@ -148,8 +155,12 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
|
||||
if(streetMap){
|
||||
set = streetMap.keySet()
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
|
||||
println city +" "+ districts.size()
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index ++
|
||||
println index +" "+ d.region
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
@@ -157,8 +168,12 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def updateRegion = null
|
||||
def flag = null
|
||||
for(def key : set){
|
||||
def alias = streetMap.get(key)
|
||||
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
|
||||
def obj = streetMap.get(key)
|
||||
def alias = null
|
||||
if(obj && obj.alias){
|
||||
alias = obj.alias
|
||||
}
|
||||
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
|
||||
def newRegion = RegionData.formatRegion(d.city, key)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
@@ -166,13 +181,26 @@ class RegionStandardBatch extends BaseBatch{
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
|
||||
if(obj.shape){
|
||||
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
|
||||
if(inAreaFlag){
|
||||
def newRegion = obj.street
|
||||
newRegion = RegionData.formatRegion(d.city, newRegion)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
}
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" ÇøÓòÐÞ¸ÄΪ£º"+updateRegion
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
if(flag!="pass"){
|
||||
def udata = new BasicDBObject()
|
||||
@@ -402,4 +430,27 @@ class RegionStandardBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def updateRegionByTown(){
|
||||
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
|
||||
def town = RegionData.formatRegion(d['B'], d['F'])
|
||||
if(town){
|
||||
|
||||
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
|
||||
if(!obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", town)
|
||||
udata.put("m_date", new Date())
|
||||
println d['F']+" "+" 区域修改为:" + town
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("status", 1)
|
||||
println d['F']+" "+" 删除:" + town
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ class RegionUpdateBatch {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionUpdateBatch()
|
||||
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchAdministrativeMappingDataBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def indexName = "administrative_mapping"
|
||||
def retMap = synchAdminiMappingTo100()
|
||||
synchToEs(retMap, "192.168.0.89", indexName)
|
||||
synchToEs(retMap, "192.168.0.52", indexName)
|
||||
println(" 运行完成")
|
||||
|
||||
}
|
||||
|
||||
def synchAdminiMappingTo100(city){
|
||||
//删除旧的,再同步新的
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
|
||||
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000)
|
||||
def datas = []
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
retMap.put(d._id.toString(), d)
|
||||
if (datas.size() >= 2000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
datas.clear();
|
||||
}
|
||||
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def synchToEs(retMap, host, indexName){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
def oldIds = new CopyOnWriteArrayList()
|
||||
|
||||
ElasticUtils89.queryUpdate(host, indexName, queryBuilder, { String id, doc->
|
||||
oldIds << id
|
||||
def mongoObj = retMap.get(id)
|
||||
def updateFieldCount = 0
|
||||
mongoObj.each{k,v->
|
||||
if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){
|
||||
def val = doc.get(k)
|
||||
if(val!=v){
|
||||
if(k!="shape" || (k=="shape" && v.type!="MultiPolygon")){
|
||||
println k +" 修改 "+id
|
||||
updateFieldCount++
|
||||
doc.put(k,v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(mongoObj && updateFieldCount){
|
||||
//进行修改
|
||||
return true
|
||||
}else{
|
||||
if(!mongoObj){
|
||||
println "删除:"+id +" "+doc.name
|
||||
ElasticUtils.delete(id, host, indexName)
|
||||
return false
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def addEsDatasMap = [:]
|
||||
retMap.each{id,map->
|
||||
if(!oldIds.contains(id)){
|
||||
println "需新增:"+id
|
||||
map.remove("c_date")
|
||||
map.remove("m_date")
|
||||
map.remove("polyline_source")
|
||||
map.remove("operator")
|
||||
map.remove("_id")
|
||||
addEsDatasMap.put(id, map)
|
||||
}
|
||||
}
|
||||
if(addEsDatasMap){
|
||||
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
/**
|
||||
* 将43上的开发区同步到mongo-100, es-89, es-52
|
||||
*/
|
||||
class SynchDevDataBatch extends BaseBatch{
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDevDataBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def indexName = "development_zone"
|
||||
def devMap = synchDevTo100()
|
||||
synchDevEs(devMap, "192.168.0.89", indexName)
|
||||
synchDevEs(devMap, "192.168.0.52", indexName)
|
||||
}
|
||||
|
||||
def synchDevTo100(){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
|
||||
def rets = []
|
||||
def devMap = [:]
|
||||
for(def r : dbcursor){
|
||||
rets << r
|
||||
devMap.put(r._id.toString(), r)
|
||||
}
|
||||
//删除旧的,再同步新的
|
||||
development_zone_CONN.remove(new BasicDBObject())
|
||||
development_zone_CONN.insert(rets)
|
||||
return devMap;
|
||||
}
|
||||
|
||||
def synchDevToEs(devMap){
|
||||
// def esDevs = findEsDevDatas()
|
||||
// def ids = []
|
||||
// for(def dev : devDatas){
|
||||
// ids << dev._id.toString()
|
||||
// def esObj = esDevs.get(dev._id.toString())
|
||||
// def updateValMap = [:]
|
||||
// if(esObj){
|
||||
// dev.each{k,v->
|
||||
// if(k!="id" && k!="_id"){
|
||||
// def val = dev.get(k)
|
||||
// if(val!=v){
|
||||
// updateValMap.put(k,val)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(!esObj || (esObj && updateValMap)){
|
||||
// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.89", "development_zone")
|
||||
// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.52", "development_zone")
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def esDelIds = []
|
||||
// esDevs.each{id,map->
|
||||
// if(!ids.contains(id)){
|
||||
// ElasticUtils.delete(id, "192.168.0.89", "development_zone")
|
||||
// ElasticUtils.delete(id, "192.168.0.52", "development_zone")
|
||||
// esDelIds << id
|
||||
// }
|
||||
// }
|
||||
// println "Es删除开发区数量:"+esDelIds.size()
|
||||
}
|
||||
|
||||
|
||||
def synchDevEs(devMap, host, indexName){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
def oldIds = new CopyOnWriteArrayList()
|
||||
|
||||
ElasticUtils89.executeQuery(host, indexName, queryBuilder, { String id, doc->
|
||||
oldIds << id
|
||||
def mongoObj = devMap.get(id)
|
||||
def updateFieldCount = 0
|
||||
mongoObj.each{k,v->
|
||||
if(!["id","_id" ,"build_date" ,"create_time","bd_shape","img_url","operator","operate_result","m_date","township","block"].contains(k)){
|
||||
def val = doc.get(k)
|
||||
if(val!=v){
|
||||
println doc.name +" 修改 "+k+" "+ v+" "+id
|
||||
updateFieldCount++
|
||||
doc.put(k,v)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(mongoObj && updateFieldCount){
|
||||
//进行修改
|
||||
return true
|
||||
}else{
|
||||
if(!mongoObj){
|
||||
println "删除:"+id +" "+doc.name
|
||||
ElasticUtils.delete(id, host, indexName)
|
||||
return false
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
def addEsDatasMap = [:]
|
||||
devMap.each{id,map->
|
||||
if(!oldIds.contains(id)){
|
||||
println "需新增:"+id
|
||||
map.remove("create_time")
|
||||
map.remove("build_date")
|
||||
map.remove("bd_shape")
|
||||
map.remove("img_url")
|
||||
map.remove("operator")
|
||||
map.remove("operate_result")
|
||||
map.remove("m_date")
|
||||
map.remove("township")
|
||||
map.remove("block")
|
||||
map.remove("_id")
|
||||
addEsDatasMap.put(id, map)
|
||||
}
|
||||
}
|
||||
if(addEsDatasMap){
|
||||
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+146
-5
@@ -1,7 +1,13 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -9,17 +15,79 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class TempExportData {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
DBCollection qianzhan_coll = MongoUtils.getCollectionByName("crawl", "yuanqu_qianzhan_com_map", "192.168.0.206", 27020)
|
||||
static main(args) {
|
||||
def task = new TempExportData()
|
||||
// task.run()
|
||||
def rets = task.fzHouseStat()
|
||||
ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
|
||||
println "文件已生成"
|
||||
// def rets = task.fzHouseStat()
|
||||
// ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
|
||||
// println "文件已生成"
|
||||
// exportDistShape()
|
||||
// task.exportDistrictAlias()
|
||||
task.exportOfficeHouseCount()
|
||||
}
|
||||
|
||||
def exportDistShape(){
|
||||
def task = new BasicDBObject()
|
||||
}
|
||||
|
||||
def exportDistrictAlias(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allRets = []
|
||||
allRets << ["Id","城市","区域","名称","地址","已删ID","已删区域","已删名称","已删地址","距离"]
|
||||
|
||||
for (def city : citys) {
|
||||
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
|
||||
def rets = findDelDistBatch(dists)
|
||||
allRets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\小区\\融合存疑小区.xlsx",allRets, "小区")
|
||||
}
|
||||
|
||||
|
||||
def findDelDistBatch(dists){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def nameAlias = dist.alias
|
||||
if(nameAlias){
|
||||
for(def nameAlia : nameAlias){
|
||||
def delDists = CommonDatautils.findAllStatusDistrict(dist.city, dist.region, nameAlia, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
|
||||
if(delDists){
|
||||
if(delDists.size()==1){
|
||||
def delDist = delDists[0]
|
||||
def distance = "缺经纬度"
|
||||
if(delDist.lat){
|
||||
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, delDist.lat/10000000000.0, delDist.lng/10000000000.0)
|
||||
}
|
||||
println totalCount.get()+" "+dists.size() +" "+dist.city
|
||||
rets << [dist._id.toString(), dist.city, dist.region, dist.name, dist.address,delDist._id.toString(), delDist.region, delDist.name, delDist.address, distance]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def run() {
|
||||
@@ -33,6 +101,8 @@ class TempExportData {
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getResults(city) {
|
||||
def rets = []
|
||||
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
|
||||
@@ -120,4 +190,75 @@ class TempExportData {
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findHouseCountByCat(def city, def region, def name, def cat){
|
||||
|
||||
String sql = "select count(*) as c from seawater where cat='${cat}' and city='${city}' and district_name='${name}' "
|
||||
if(region){
|
||||
sql = sql + " and region='${region}'"
|
||||
}
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
def findYuanQu(city, name){
|
||||
def q = new BasicDBObject("name", Pattern.compile("^${name}"))
|
||||
def dbcursor = qianzhan_coll.find(q)
|
||||
def obj = null
|
||||
for(def d : dbcursor){
|
||||
if(d.city.contains(city)){
|
||||
obj = d
|
||||
break;
|
||||
}
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
def exportOfficeHouseCount(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
|
||||
// def obj = findYuanQu(d['B'], d['D'])
|
||||
// if(obj){
|
||||
// d['T'] = "前瞻存在"
|
||||
// }
|
||||
// if(!d['T']){
|
||||
// def city = RegionData.formatCity(d['C'])
|
||||
// def obj = findYuanQu(city, d['E'])
|
||||
// if(obj){
|
||||
// d['T'] = "前瞻存在"
|
||||
// d['U'] = obj.name
|
||||
// }
|
||||
// }
|
||||
// if(!d['T']) {
|
||||
// def city = RegionData.formatCity(d['C'])
|
||||
// def region = RegionData.formatRegion(city, d['D'])
|
||||
//
|
||||
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
|
||||
// if (!count) {
|
||||
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
|
||||
// }
|
||||
// d['T'] = count
|
||||
// }
|
||||
|
||||
if(!d['T']){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
|
||||
if(datas){
|
||||
def sources = []
|
||||
for(def o : datas){
|
||||
sources << o.source
|
||||
}
|
||||
if(sources){
|
||||
d['V'] = sources.join(",")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
+52
-100
@@ -1,105 +1,57 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class Test{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
def task = new Test()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def i=1
|
||||
while(i<2){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
def rets = process(districts)
|
||||
println i + " "+districts.size() +" "+rets.size()
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def process(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
methodA(rMap)
|
||||
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
|
||||
rets << rMap.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def methodA(rMap) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
// 模拟一个可能会抛出异常的操作
|
||||
riskyOperation();
|
||||
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
|
||||
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(10000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
|
||||
}
|
||||
}
|
||||
|
||||
// 一个可能会抛出异常的操作
|
||||
def riskyOperation() throws Exception {
|
||||
// 模拟一个有50%概率抛出异常的操作
|
||||
if (Math.random() < 0.5) {
|
||||
throw new Exception("Simulated exception");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
|
||||
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
|
||||
Date creationDate = objectId.getDate();
|
||||
System.out.println(creationDate);
|
||||
|
||||
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
|
||||
//
|
||||
//
|
||||
//def city = "昌江"
|
||||
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
|
||||
// def date = d['B']
|
||||
// def start = DateUtils.getFirstDayOfMonth(date)
|
||||
// def end = DateUtils.getLastDayOfMonth(date)
|
||||
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
|
||||
// for(def weekend : weekends){
|
||||
// def avg_price = d['I']
|
||||
// def val = random(-200,200)
|
||||
// def new_price = (int)(avg_price*val/10000 + avg_price)
|
||||
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
|
||||
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
|
||||
//
|
||||
// def o = new BasicDBObject()
|
||||
// o.put("avg_price", new_price)
|
||||
// o.put("c_date" , new Date())
|
||||
// o.put("city", city)
|
||||
// o.put("m_date" , new Date())
|
||||
// o.put("weekend", weekend)
|
||||
// o.put("cat","城市")
|
||||
// o.put("moving_average" , (int)avg_price)
|
||||
// if(!last_price){
|
||||
// o.put("weekend_markup" , 0)
|
||||
// }else{
|
||||
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
|
||||
// o.put("weekend_markup" , weekend_markup)
|
||||
// }
|
||||
//
|
||||
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
|
||||
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
|
||||
// }
|
||||
//},1)
|
||||
//
|
||||
//
|
||||
//public static final int random(int i1, int i2) {
|
||||
// int max = Math.max(i1, i2);
|
||||
// int min = Math.min(i1, i2);
|
||||
// return min + RandomUtils.nextInt(max - min + 1);
|
||||
//}
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
-6
@@ -52,9 +52,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 206){
|
||||
continue
|
||||
}
|
||||
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
|
||||
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||
matchByCity(c, noMatchDatas)
|
||||
@@ -123,9 +120,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 70){
|
||||
continue
|
||||
}
|
||||
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
|
||||
def addCount = addDatasToDB(crawlDatas)
|
||||
println c + " ÐÂÔö£º"+ addCount
|
||||
|
||||
+104
-11
@@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
@@ -45,6 +44,8 @@ class TownMiddleProcess {
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
@@ -64,7 +65,7 @@ class TownMiddleProcess {
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
// synchTown()
|
||||
synchTown()
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
@@ -95,7 +96,13 @@ class TownMiddleProcess {
|
||||
// addTownCenter()//百度接口添加中心点
|
||||
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
|
||||
|
||||
addSpecialCityTownAlias()
|
||||
// addSpecialCityTownAlias()
|
||||
|
||||
|
||||
// updateNewRegionPoyline()//更新新的区域围栏
|
||||
|
||||
updateMappingPolyline()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -451,16 +458,16 @@ class TownMiddleProcess {
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
if (datas.size() >= 2000) {
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
@@ -681,7 +688,7 @@ class TownMiddleProcess {
|
||||
}
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
@@ -1356,4 +1363,90 @@ class TownMiddleProcess {
|
||||
keys.put("street", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
|
||||
}
|
||||
|
||||
def updateNewRegionPoyline(){
|
||||
|
||||
def regionPolylineMap = regionPolyline()
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
// q.put("city", "苏州市")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
for(def d : dbcursor){
|
||||
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
|
||||
println d.city+" "+d.district
|
||||
if(shapeData){
|
||||
println "新围栏数:"+shapeData.shape.coordinates.size()
|
||||
}
|
||||
|
||||
if(d.shape){
|
||||
println "旧围栏数:"+d.shape.coordinates.size()
|
||||
}
|
||||
|
||||
if(d.shape && shapeData && shapeData.shape!=d.shape){
|
||||
def udata = new BasicDBObject()
|
||||
udata.shape = shapeData.shape
|
||||
udata.location = shapeData.location
|
||||
udata.m_date = new Date()
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
|
||||
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
|
||||
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
|
||||
// if(Math.abs(area1-area)>=1){
|
||||
// println "新:"+area
|
||||
// println "旧:"+area1
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.shape = shapeData.shape
|
||||
// udata.location = shapeData.location
|
||||
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateMappingPolyline(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("shape.type","polygon")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
updateAdministrativeMapping(d._id.toString(), d.shape)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateAdministrativeMapping(def id, def shape){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.shape = params.newshape",
|
||||
"params":{
|
||||
"newshape":{
|
||||
"type":"${shape.type}",
|
||||
"coordinates":${shape.coordinates}
|
||||
}
|
||||
}
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"_id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+569
-68
@@ -1,17 +1,36 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
@@ -22,6 +41,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def updateList = []
|
||||
def addAliasList = []
|
||||
@@ -37,12 +57,17 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def TownStandard(){
|
||||
// townStandardByCat("office")
|
||||
townStandardByPoi()
|
||||
townStandardByCat("office")
|
||||
// townStandardByPoi()
|
||||
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
|
||||
// standardTown()
|
||||
// updateTownData()
|
||||
|
||||
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
|
||||
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
|
||||
def citys = ["天津"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
@@ -53,10 +78,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
|
||||
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias","shape"], mzbRegionMap)
|
||||
if(townMap){
|
||||
def regionSet = townMap.keySet()
|
||||
if(regionSet.size()>1){
|
||||
if(regionSet.size()>=1){
|
||||
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
|
||||
}else{
|
||||
println city +" 无区域"
|
||||
@@ -69,11 +94,15 @@ class TownStandardBatch extends BaseBatch{
|
||||
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||
|
||||
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
|
||||
//浙江,江苏,安徽
|
||||
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
|
||||
def runProvinces = ["福建省","浙江省","江苏省","安徽省","山东省","甘肃省"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
|
||||
index++
|
||||
def data = fullCityMap.get(city)
|
||||
// if(runProvinces.contains(data.province)){
|
||||
// continue
|
||||
// }
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
@@ -95,11 +124,16 @@ class TownStandardBatch extends BaseBatch{
|
||||
def rets = readPoiCsv(fileName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
if(updateList.size()>50000){
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-${index}.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名-${index}.xlsx", addAliasList, "数据")
|
||||
updateList.clear()
|
||||
}
|
||||
}else{
|
||||
println fileName+" 文件没找到"
|
||||
}
|
||||
@@ -107,18 +141,18 @@ class TownStandardBatch extends BaseBatch{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-最后.xlsx", updateList, "数据")
|
||||
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township","lng","lat"])
|
||||
for(def d : districts){
|
||||
if(d.township){
|
||||
def towns = null
|
||||
def updateTown = null
|
||||
|
||||
def towns = null
|
||||
def updateTown = null
|
||||
if(d.region){
|
||||
for(def m : townMap){
|
||||
def fullRegionName = shortRegionNameMap.get(d.region)
|
||||
if(m.key == fullRegionName){
|
||||
@@ -126,41 +160,51 @@ class TownStandardBatch extends BaseBatch{
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!towns){
|
||||
towns = townMap.get(fullCityName)
|
||||
}
|
||||
|
||||
def opt = true
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
def townAlias = []
|
||||
if(town.alias){
|
||||
townAlias = town.alias
|
||||
}
|
||||
townAlias << town.street
|
||||
if(townAlias.contains(d.township)){
|
||||
if(d.township!=town.street){
|
||||
updateTown = town.street
|
||||
}else{
|
||||
opt = false
|
||||
}
|
||||
break
|
||||
}
|
||||
def opt = true
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
def townAlias = []
|
||||
if(town.alias){
|
||||
townAlias = town.alias
|
||||
}
|
||||
}
|
||||
if(opt){
|
||||
if(updateTown){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", updateTown)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
townAlias << town.street
|
||||
if(townAlias.contains(d.township)){
|
||||
if(d.township!=town.street){
|
||||
updateTown = town.street
|
||||
}else{
|
||||
opt = false
|
||||
}
|
||||
break
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", "")
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
if(town.shape){
|
||||
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
updateTown = town.street
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(opt){
|
||||
if(updateTown){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", updateTown)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", "")
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -168,7 +212,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
for(def d : rets){
|
||||
@@ -221,21 +265,21 @@ class TownStandardBatch extends BaseBatch{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":updateTown])
|
||||
results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
updateList << [d.id, d.province, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":""])
|
||||
// results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
updateList << [d.id, d.province, d.city, d.district, d.name, d.township, d.status, ""]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
|
||||
}
|
||||
}
|
||||
@@ -244,10 +288,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
// if(results.size()>0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
if(results.size()>0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println fullCityName +" 处理完成"
|
||||
}
|
||||
|
||||
@@ -322,18 +366,475 @@ class TownStandardBatch extends BaseBatch{
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
try{
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def readCSV3(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
def index = 0
|
||||
for(line in lines) {
|
||||
if(line && index>=1){
|
||||
String[] arrs = line.split(",")
|
||||
if(arrs.size()>=11 && arrs[10]=="是" && arrs[6]!=arrs[7]){
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("id", arrs[0])
|
||||
o.put("township", arrs[7])
|
||||
o.put("city", arrs[2])
|
||||
o.put("old_district", arrs[3])
|
||||
o.put("new_district", arrs[8])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
index++
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def readCSV2(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
def index = 0
|
||||
for(line in lines) {
|
||||
if(line && index>=1){
|
||||
String[] arrs = line.split(",")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("id", arrs[1])
|
||||
o.put("province", arrs[2])
|
||||
o.put("city", arrs[3])
|
||||
o.put("district", arrs[4])
|
||||
o.put("lng", arrs[5])
|
||||
o.put("lat", arrs[6])
|
||||
o.put("data_source", arrs[7])
|
||||
if(arrs.size()>=9){
|
||||
o.put("township", arrs[8])
|
||||
o.put("new_district", arrs[9])
|
||||
o.put("town_source", arrs[10])
|
||||
}
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
}
|
||||
index++
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addTownByTownShape(){
|
||||
def i = 0
|
||||
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
|
||||
while(i<=50){
|
||||
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\no_township_${i}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()) {
|
||||
def rets = readCSV2(fileName)
|
||||
println i +" "+ rets.size()
|
||||
def rows = addTownshipBatch(rets)
|
||||
if (CommonDatautils.isWindow()) {
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}else{
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}
|
||||
println fileName +" 运行完成"
|
||||
}else{
|
||||
println fileName +" 不存在"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def standardTown(){
|
||||
def i = 49
|
||||
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
|
||||
|
||||
def cityTownMap = CommonDatautils.queryAllTown(["city", "district", "mzb_district_name", "street", "alias"])
|
||||
while(i<=50){
|
||||
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_result_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()) {
|
||||
def rets = readCSV2(fileName)
|
||||
println i +" "+ rets.size()
|
||||
def rows = addTownshipBatch2(rets, cityTownMap)
|
||||
if (CommonDatautils.isWindow()) {
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}else{
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}
|
||||
println fileName +" 运行完成"
|
||||
}else{
|
||||
println fileName +" 不存在"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
def addTownshipBatch(def rets){
|
||||
def results = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def r = rets[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def rMap = regionFullNameMap.get(r.city)
|
||||
if(!rMap){
|
||||
def data = fullCityMap.get(r.city)
|
||||
rMap = regionFullNameMap.get(data.city)
|
||||
}
|
||||
if (rMap) {
|
||||
|
||||
// def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
|
||||
// def set = rMap.keySet()
|
||||
println totalCount.get() +" "+r.city + " "
|
||||
// if (set.size() >= 1) {
|
||||
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
|
||||
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
|
||||
// }
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
def addTownshipBatch2(def rets, def cityTownMap){
|
||||
def results = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def r = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def rMap = regionFullNameMap.get(r.city)
|
||||
def fullcity = null
|
||||
if(!rMap){
|
||||
def data = fullCityMap.get(r.city)
|
||||
rMap = regionFullNameMap.get(data.city)
|
||||
fullcity = data.city
|
||||
}
|
||||
if (rMap) {
|
||||
def townMap = cityTownMap.get(fullcity)
|
||||
def set = rMap.keySet()
|
||||
if (set.size() >= 1) {
|
||||
if(townMap.size()>=1){
|
||||
r.strand_town = isStandTown(townMap, r.new_district, r.township)
|
||||
}
|
||||
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return results
|
||||
}
|
||||
|
||||
def addTown(saveObj, townMap, location) {
|
||||
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
|
||||
// def towns = townMap.get(saveObj.district)
|
||||
// if (towns) {
|
||||
// for (def town : towns) {
|
||||
// if (town.shape && town.shape.coordinates) {
|
||||
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
// if (flag) {
|
||||
// saveObj.township = town.street
|
||||
// saveObj.town_source = "同区域"
|
||||
// saveObj.strand_town = "是"
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def townNames = townMap.keySet()
|
||||
// for(def townName : townNames){
|
||||
// towns = townMap.get(townName)
|
||||
// for (def town : towns) {
|
||||
// if (town.shape && town.shape.coordinates) {
|
||||
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
// if (flag) {
|
||||
// saveObj.township = town.street
|
||||
// saveObj.new_district = town.mzb_district_name
|
||||
// saveObj.town_source = "忽略区域"
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(saveObj.township){
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//通过调取地图api接口获取乡镇街道
|
||||
if(!saveObj.township){
|
||||
def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
saveObj.township = gaodeMap.township
|
||||
saveObj.new_district = gaodeMap.district
|
||||
saveObj.town_source = "高德API"
|
||||
saveObj.strand_town = ""
|
||||
// if(townMap.size()>=1){
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
// //通过调取地图api接口获取乡镇街道
|
||||
// if(!saveObj.township){
|
||||
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
|
||||
// if(gaodeMap && gaodeMap.addressComponent) {
|
||||
// def addressComponent = gaodeMap.addressComponent
|
||||
// if(addressComponent){
|
||||
// if(addressComponent.town){
|
||||
// saveObj.township = addressComponent.town
|
||||
// saveObj.new_district = addressComponent.district
|
||||
// saveObj.town_source = "百度API"
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def isStandTown(townMap, district, town){
|
||||
def towns = townMap.get(district)
|
||||
def flag = "非"
|
||||
if(towns){
|
||||
for(def t : towns){
|
||||
if(t.street == town){
|
||||
flag = "是"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
def jsonObject = null
|
||||
// while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
city = addressComponent.city
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(jsonObject.status==0 || count>3){
|
||||
// stop = true
|
||||
// println "高德调用超3次"
|
||||
// }else{
|
||||
// count ++
|
||||
// Thread.sleep(3000)
|
||||
// print "."
|
||||
// }
|
||||
// }
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateTownData(){
|
||||
def i = 49
|
||||
while(i<50) {
|
||||
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv"
|
||||
}
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
def rets = readCSV3(fileName)
|
||||
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
|
||||
if(rets){
|
||||
// updateToEs(rets)//不能用这种方法修改,否则其它字段都将丢失
|
||||
// updateToEs2(rets)
|
||||
updateDistTownship(rets)
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.containsKey("way")){
|
||||
map.remove("way")
|
||||
}
|
||||
request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
def updateToEs2(rows) {
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id, def township){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/map_poi_2024/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.township='${township}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"_id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updateDistTownship(rets){
|
||||
for(def m : rets){
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(m.id)
|
||||
if(obj.township != m.township){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", m.township)
|
||||
if(!m.new_district.contains(m.old_district)){
|
||||
def lists = CommonDatautils.findAllDistrictByName(obj.city, null, obj.name, obj.cat)
|
||||
if(lists.size()<2){
|
||||
continue
|
||||
}
|
||||
def region = RegionData.formatRegion(m.city, m.new_district)
|
||||
if(region){
|
||||
def dbo = CommonDatautils.findDistrict(m.city, region, obj.name, obj.cat)
|
||||
if(dbo){
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
|
||||
println "删除 "+m.city +" "+obj.name+" "+m.id+" "+obj.cat
|
||||
continue
|
||||
}else{
|
||||
udata.put("region", region)
|
||||
}
|
||||
}
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+72
-2
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
@@ -11,6 +12,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -42,13 +44,16 @@ class TownshipStrandUpdate {
|
||||
// task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
task.updateDiffRegionTown()
|
||||
// task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
// task.updateDevMapping()
|
||||
println new Date().getTime()
|
||||
// println new Date().getTime()
|
||||
// task.updateSpecCityTown()
|
||||
|
||||
|
||||
task.updateTownshipAlias()
|
||||
}
|
||||
|
||||
|
||||
@@ -629,4 +634,69 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def updateTownshipAlias(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\20250219--乡镇街道添加别名.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def q = new BasicDBObject("type","乡镇街道").append("city", d["A"]).append("district",d["B"]).append("street",d["C"])
|
||||
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
|
||||
def newAlias = d['D']
|
||||
if(list.size()==0){
|
||||
list = q.put("street", d['D'])
|
||||
newAlias = d['C']
|
||||
list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
|
||||
}
|
||||
if(list && list.size()==1){
|
||||
def obj = list[0]
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(newAlias)){
|
||||
alias << newAlias
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("operator","陈杰")
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
// updateStreetAlias(obj._id.toString(), alias.join("','"))
|
||||
}else{
|
||||
println d["C"] +" "+ d["D"]
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def updateStreetAlias(def id, def alias){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.alias=['${alias}']",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"_id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class VillaHouseBatch extends BaseBatch{
|
||||
|
||||
def pgsqlInstance236 = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new VillaHouseBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city", "region", "name", "estate_type2"], "all")
|
||||
def villaDists = findVillaDists(dists)
|
||||
addVillaTypeFlag(villaDists)
|
||||
println "运行完成"
|
||||
}
|
||||
|
||||
def findVillaDists(dists){
|
||||
def villaDists = []
|
||||
for(def d : dists){
|
||||
if(d.estate_type2 == "别墅"){
|
||||
villaDists << d
|
||||
}
|
||||
}
|
||||
return villaDists
|
||||
}
|
||||
|
||||
def addVillaTypeFlag(def villaDists){
|
||||
for(def i=0;i<villaDists.size();i++){
|
||||
def dist = villaDists[i]
|
||||
|
||||
def sellIds = findHouseIdByCommId(dist._id.toString(), "310000")
|
||||
println dist._id.toString() +" "+sellIds.size()
|
||||
if(sellIds){
|
||||
def retMap = run(sellIds)
|
||||
if(retMap){
|
||||
println "最终修改:"+retMap.size()
|
||||
updateHouseFlag(retMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(sellIds){
|
||||
def retMap = new ConcurrentHashMap()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(sellIds.size())
|
||||
for(def i=0;i<sellIds.size();i++){
|
||||
final def houseId = sellIds[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
retMap = findHouseInfo(houseId, retMap)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findHouseIdByCommId(commid, cityId){
|
||||
def sql = "select gujia_sell_id from base.bigdata_forsale where urbn_id='${cityId}' and gujia_comm_id='${commid}'";
|
||||
def sellIds = []
|
||||
sqlInstance235.eachRow(sql,{r->
|
||||
sellIds << r.gujia_sell_id
|
||||
})
|
||||
return sellIds
|
||||
}
|
||||
|
||||
def findHouseInfo(sell_id, houseMap){
|
||||
def flag = findHouseDetail(sell_id)
|
||||
if(flag){
|
||||
houseMap.put(sell_id, flag)
|
||||
}
|
||||
return houseMap
|
||||
}
|
||||
|
||||
def findHouseDetail(houseId){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", houseId))
|
||||
def flag = null
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(5);
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def house = houses[k]
|
||||
def title = house.name
|
||||
println title
|
||||
if(title){
|
||||
if(title.contains("独栋")){
|
||||
flag = "独栋"
|
||||
}else if(title.contains("联排")){
|
||||
flag = "联排"
|
||||
}else if(title.contains("叠拼")){
|
||||
flag = "叠拼"
|
||||
}else if(title.contains("花园洋房")){
|
||||
flag = "花园洋房"
|
||||
}else if(title.contains("双拼")){
|
||||
flag = "双拼"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
def updateHouseFlag(retMap){
|
||||
retMap.each{sell_id,type->
|
||||
def sql = "update base.bigdata_forsale set villa_type='${type}' where gujia_sell_id='${sell_id}'"
|
||||
sqlInstance235.executeUpdate(sql)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+144
-21
@@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
||||
|
||||
//
|
||||
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
||||
def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房",
|
||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
|
||||
|
||||
@@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
|
||||
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
|
||||
final static def EMAILS = [
|
||||
"gaojuan@fangjia.com"
|
||||
@@ -68,6 +71,38 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictFieldAddBatch(), args)
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
|
||||
//// if(dist){
|
||||
//// d['L'] = dist.loudong_count
|
||||
//// d['M'] = dist.household_count
|
||||
//// d['Q'] = dist.estate_type2
|
||||
//// }
|
||||
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
|
||||
// if(!list && dist){
|
||||
// def alias = dist.alias
|
||||
// for(def name : alias){
|
||||
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
|
||||
// if(list){
|
||||
// d['Z'] = "别名"
|
||||
// }
|
||||
// }
|
||||
// }else{
|
||||
// d['Z'] = "名称"
|
||||
// }
|
||||
// if(list){
|
||||
// def obj = list[0]
|
||||
// if(obj.loudong_cnt){
|
||||
// d['W'] = obj.loudong_cnt
|
||||
// }
|
||||
//
|
||||
// if(obj.households){
|
||||
// d['X'] = obj.households
|
||||
// }
|
||||
// d['Y'] = obj.url
|
||||
// }
|
||||
//
|
||||
// })
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -83,18 +118,13 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
headers << fieldsNameMap.get(it)
|
||||
}
|
||||
|
||||
headers << "抓取量"
|
||||
headers << "匹配量"
|
||||
headers << "0.9分以上量"
|
||||
headers << "匹配率"
|
||||
headers << "0.9以上匹配率"
|
||||
// analysRets << headers
|
||||
|
||||
for(def c : citys){
|
||||
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||
if(totalCount){
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
||||
// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
||||
analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表
|
||||
}
|
||||
}
|
||||
println analysRets.toString()
|
||||
@@ -107,8 +137,10 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findEsDatas(c, esWebSource)
|
||||
|
||||
def matchRets1 = matchApi(esRets)
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
|
||||
@@ -140,6 +172,65 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def addFieldFromMatchMapping(c, totalCount, analysRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findMatchSameDatas(c, esWebSource)
|
||||
println esWebSource +" 新老算法匹配相同量:"+esRets.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , esRets, esWebSource)
|
||||
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
|
||||
def analyRet = []
|
||||
analyRet << seaweedmapSource
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analysRets << analyRet
|
||||
}
|
||||
return analysRets;
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findMatchSameDatas(c, esWebSource){
|
||||
def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true))
|
||||
def siteDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||
def rows = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(siteDatas.size())
|
||||
for(def i=0;i<siteDatas.size();i++){
|
||||
final def d = siteDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findCommunities(d.url)
|
||||
if(obj){
|
||||
obj.match_id = d.match_id
|
||||
rows << obj
|
||||
}
|
||||
// println d.city +" "+totalCount.get() +"/"+siteDatas.size()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addFieldFromRonghe(c, totalCount, rh_analysRets){
|
||||
@@ -209,14 +300,10 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
fields << "city"
|
||||
fields << "estate_type2"
|
||||
def fieldValueMap = [:]
|
||||
def rets = []
|
||||
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
|
||||
for(def matchDist : matchRets){
|
||||
|
||||
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
|
||||
// continue
|
||||
// }
|
||||
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
println c+" "+matchDist.match_id+" 不存在"
|
||||
@@ -224,7 +311,6 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
def newValues = strandField(c, dist, matchDist)
|
||||
if(newValues){
|
||||
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
@@ -244,7 +330,9 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
if(newVal) {
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
@@ -312,12 +400,16 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
if(seaweedfield == "lng"){
|
||||
def locationMap = FiledStrand.processLngAndLat(esVal)
|
||||
if(locationMap){
|
||||
newValues.put("lat", locationMap.lat)
|
||||
newValues.put("lng", locationMap.lng)
|
||||
newValues.put("lat2", locationMap.lat2)
|
||||
newValues.put("lng2", locationMap.lng2)
|
||||
try{
|
||||
def locationMap = FiledStrand.processLngAndLat(esVal)
|
||||
if(locationMap){
|
||||
newValues.put("lat", locationMap.lat)
|
||||
newValues.put("lng", locationMap.lng)
|
||||
newValues.put("lat2", locationMap.lat2)
|
||||
newValues.put("lng2", locationMap.lng2)
|
||||
}
|
||||
}catch(Exception e){
|
||||
println esVal
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -361,6 +453,15 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
return list
|
||||
}
|
||||
|
||||
|
||||
static def findEsDatasByCityAndName(c, district, name, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' and region='${district}' and name='${name}'", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
@@ -621,4 +722,26 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
|
||||
}
|
||||
}
|
||||
|
||||
def findCommunities(url){
|
||||
if(!url){
|
||||
return null
|
||||
}
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||
def rets = []
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
}catch(Exception e){
|
||||
findCommunities(url)
|
||||
}
|
||||
def obj = null
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
obj = d
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictStatRentCleanBatch extends BaseBatch{
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictStatRentCleanBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def keys = CommonDatautils.getSimpleDistricts("district", c)
|
||||
running(keys, c)
|
||||
println c +" ÔËÐÐÍê³É"
|
||||
}
|
||||
}
|
||||
|
||||
def running(keys, c){
|
||||
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
|
||||
def fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("district_name", 1)
|
||||
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
|
||||
def delRets = []
|
||||
for(def d : dbcursor){
|
||||
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
|
||||
println d.city+"@"+d.region+"@"+d.district_name
|
||||
delRets << d._id
|
||||
}
|
||||
if(delRets.size()>=100){
|
||||
def delquery = new BasicDBObject()
|
||||
delquery.put("_id", new BasicDBObject('$in', delRets))
|
||||
RENT_DIST_STAT_COLL.remove(delquery)
|
||||
delRets.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+30
-9
@@ -1,16 +1,9 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.addrepo.InvokeHttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.HttpRequestUtil
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FangjiaClientUtils {
|
||||
|
||||
@@ -336,7 +329,7 @@ class FangjiaClientUtils {
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=true&addressAssessment=false"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
@@ -354,7 +347,7 @@ class FangjiaClientUtils {
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
def url = null
|
||||
address = address.replace("{","")
|
||||
address = address.toString().replace("{","")
|
||||
int retryCount = 0
|
||||
while (retryCount < 3) {
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
|
||||
@@ -398,4 +391,32 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 经纬度转换
|
||||
* 1、高德转百度 2、高德转WGS84 3、百度转高德 4、WGS84转高德 5、百度转WGS84 6、WGS84转百度
|
||||
*/
|
||||
public static def convert(String locationstr, type){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
def points = []
|
||||
try {
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/convert?locations="+URLEncoder.encode(locationstr,"UTF-8")+"&coordsys=${type}"
|
||||
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
if(jsonObj && jsonObj.locations){
|
||||
def locations = jsonObj.locations
|
||||
for(def s : locations){
|
||||
def arrs = s.split(",")
|
||||
points = [arrs[0].toString().toDouble(), arrs[1].toString().toDouble()]
|
||||
}
|
||||
}
|
||||
return points
|
||||
} catch (Exception e) {
|
||||
println e.getMessage()
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 写字楼字段填充入库(自动入库)
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class OfficeFieldAddBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def sources = ["安居客app","办办网","房天下","中华写字楼", "点点租"]
|
||||
|
||||
//type:租售类别 wall:外墙
|
||||
// def fieldsMap = ["elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","block":"block",
|
||||
// "estate_type2":"property_type","estate_company":"property_company","estate_charge":"property_fee","all_area":"area",
|
||||
// "standard_area":"standard_area","sight":"open_area","build_area":"build_area","build_type":"build_type","total_floor":"total_floor",
|
||||
// "finish_building_date":"complete_time","loop_line_info":"ring_location","standard_height":"floor_height",
|
||||
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","developer":"developer","fitment":"decoration","parking":"parking_cnt",
|
||||
// "feature":"characteristic","description":"introduction","parking_fee":"car_fee","property_life":"property_life","bed_kind":"level",
|
||||
// "lobby_height":"lobby_height","clear_height":"clear_height","foreign":"foreign","build_structure":"structure","address":"address",
|
||||
// "lng":"location","open_date":"open_date","live_date":"live_date","real_use_percent":"room_rate","land_car_site":"land_car_site",
|
||||
// "underground_car_site":"underground_car_site","land_floor":"land_floor","underground_floor":"underground_floor"]
|
||||
|
||||
def fieldsMap = ["build_area":"build_area"]
|
||||
|
||||
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","estate_charge":"property_fee","finish_building_date":"complete_time",
|
||||
// "bed_kind":"level","land_car_site":"land_car_site","underground_car_site":"underground_car_site","parking_fee":"car_fee",
|
||||
// "elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","clear_height":"clear_height",
|
||||
// "foreign":"foreign","standard_area":"standard_area","foreign":"foreign","parking_fee":"car_fee","estate_company":"property_company","developer":"developer","all_area":"area",
|
||||
|
||||
static main(args) {
|
||||
// runBatch(new OfficeFieldAddBatch(), args)
|
||||
|
||||
def task = new OfficeFieldAddBatch()
|
||||
// task.fieldTempProcess()
|
||||
task.deleteNotNormalOffice()
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = ["深圳"]//CommonDatautils.findAllCity()
|
||||
def fields = fieldsMap.keySet() as List
|
||||
def analysRets = []
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
|
||||
|
||||
sources.each {source->
|
||||
def esRets = findAllEsDataByCity(c, source)
|
||||
def updateRets = processFieldsFromMatch(c , esRets, source)
|
||||
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
|
||||
def analyRet = []
|
||||
analyRet << source
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analysRets << analyRet
|
||||
}
|
||||
}
|
||||
println analysRets.toString()
|
||||
}
|
||||
|
||||
def processFieldsFromMatch(c,matchRets, source){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
def fieldValueMap = [:]
|
||||
for(def esOffice : matchRets){
|
||||
// if(esOffice.name!="东方建富时尚广场"){
|
||||
// continue
|
||||
// }
|
||||
def office = CommonDatautils.findNormalDistrict(esOffice.city, esOffice.region, esOffice.name, "office", fields)
|
||||
if(!office){
|
||||
// println c+" "+esOffice.name+" 不存在"
|
||||
continue
|
||||
}
|
||||
def newValues = strandField(c, office, esOffice, source)
|
||||
if(newValues){
|
||||
println office._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(office._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
// ["loop_line_info":"ring_location","feature":"characteristic","description":"introduction","lng":"location","open_date":"open_date"]
|
||||
def strandField( c, office, esOffice, source){
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = office.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = esOffice.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
|
||||
if(seaweedfield == "estate_type2"){
|
||||
newVal = FiledStrand.processEstateType(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), office.estate_type2)
|
||||
if(newVal) {
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
if(seaweedfield =="virescence_percent"){
|
||||
newVal = FiledStrand.processVirescencePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "loop_line_info"){
|
||||
newVal = FiledStrand.processLoopLine(esVal)
|
||||
}
|
||||
if(seaweedfield =="cubage_percent"){
|
||||
newVal = FiledStrand.processCubagePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area" || seaweedfield=="sight"){
|
||||
newVal = FiledStrand.processArea(esVal.toString())
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
|
||||
if(seaweedfield =="car_site_count" || seaweedfield =="land_car_site" || seaweedfield =="underground_car_site"){
|
||||
newVal = FiledStrand.processCarSite(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="property_life"){
|
||||
newVal = FiledStrand.processPropertyLife(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="address"){
|
||||
newVal = FiledStrand.processAddress(esVal, office.city)
|
||||
}
|
||||
if(seaweedfield =="block"){
|
||||
newVal = FiledStrand.processBlock(office.city, office.region, esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "fitment"){
|
||||
newVal = FiledStrand.processFitment(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_type"){
|
||||
newVal = FiledStrand.processBuildType(esVal, "office")
|
||||
}
|
||||
if(seaweedfield == "build_structure"){
|
||||
newVal = FiledStrand.processBuildStructure(esVal)
|
||||
}
|
||||
if(seaweedfield == "bed_kind"){
|
||||
newVal = FiledStrand.processLevel(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "total_floor" || seaweedfield == "land_floor"){
|
||||
newVal = FiledStrand.processFloor(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "underground_floor"){
|
||||
newVal = FiledStrand.processUndergroundFloor(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "real_use_percent"){
|
||||
newVal = FiledStrand.processRealUsePercent(esVal)
|
||||
}
|
||||
if(seaweedfield == "parking_fee"){
|
||||
newVal = FiledStrand.processCarSiteRent(esVal)
|
||||
println "[ ${source} ] old: "+esVal +" new:"+newVal
|
||||
}
|
||||
|
||||
if(seaweedfield == "elevators_count"){
|
||||
newVal = FiledStrand.processElevators(esVal)
|
||||
}
|
||||
if(seaweedfield == "person_elevators_count" && esVal.contains("客梯")){
|
||||
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("客梯")))
|
||||
}
|
||||
|
||||
if(seaweedfield == "goods_elevator_count" && esVal.contains("货梯")){
|
||||
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("货梯")))
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_height" || seaweedfield=="lobby_height" || seaweedfield=="clear_height"){
|
||||
newVal = FiledStrand.processStandardHeight(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_area"){
|
||||
newVal = FiledStrand.processStandardArea(esVal.toString())
|
||||
}
|
||||
|
||||
if(seaweedfield == "finish_building_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "foreign"){
|
||||
newVal = FiledStrand.processForeign(esVal)
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
if(seaweedfield == "lng"){
|
||||
try{
|
||||
def locationMap = FiledStrand.processLngAndLat(esVal)
|
||||
if(locationMap){
|
||||
newValues.put("lat", locationMap.lat)
|
||||
newValues.put("lng", locationMap.lng)
|
||||
newValues.put("lat2", locationMap.lat2)
|
||||
newValues.put("lng2", locationMap.lng2)
|
||||
}
|
||||
}catch(Exception e){
|
||||
println esVal
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return newValues;
|
||||
}
|
||||
|
||||
|
||||
def findAllEsDataByCity(city, source){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", source))
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "office2", queryBuilder);
|
||||
}catch(Exception e){
|
||||
datas = findAllEsDataByCity(city, source)
|
||||
}
|
||||
|
||||
return datas;
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
|
||||
for(def rm : updateRets){
|
||||
def valMap = rm.value
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
if(valMap.containsKey(seaweedfield)) {
|
||||
if(countMap.containsKey(seaweedfield)) {
|
||||
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
|
||||
}else{
|
||||
countMap.put(seaweedfield, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println countMap.toString()
|
||||
def precentMap = [:]
|
||||
fieldsMap.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
|
||||
}else {
|
||||
precentMap.put(key, 0)
|
||||
}
|
||||
}
|
||||
return precentMap;
|
||||
}
|
||||
|
||||
def updateDB(c, updateRets){
|
||||
|
||||
for(def rm : updateRets) {
|
||||
def id = rm.key
|
||||
def valMap = rm.value
|
||||
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
valMap.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def fieldTempProcess(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx",{d->
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("address", d['F'])
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
|
||||
})
|
||||
}
|
||||
|
||||
def deleteNotNormalOffice(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\需删除写字楼.xlsx",{d->
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("status", 1)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
|
||||
})
|
||||
}
|
||||
}
|
||||
+179
-26
@@ -74,11 +74,11 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
// addBlock()
|
||||
// addFieldValue()
|
||||
// updateErrorField()
|
||||
updateErrorField()
|
||||
// export()
|
||||
// test()
|
||||
// addOffice()
|
||||
fileAddBlock()
|
||||
// fileAddBlock()
|
||||
}
|
||||
|
||||
def export(){
|
||||
@@ -134,33 +134,196 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def updateErrorField(){
|
||||
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
|
||||
def offices = CommonDatautils.findAllDistricts("ÉϺ£","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
|
||||
def offices = CommonDatautils.findAllDistrict("office", ["underground_car_site"])
|
||||
// def offices = CommonDatautils.findAllDistricts("上海","office", ["build_area"])
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
|
||||
for(def i=0;i<offices.size();i++){
|
||||
final def o = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
if(o._id.toString()=="5954d66f9aed7b45d3785a40"){
|
||||
println o._id.toString()
|
||||
}
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
updateDistField(o)
|
||||
println totalCount.get() +" "+o.city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def updateDistField(o){
|
||||
def addr_alias = o.addr_alias
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
|
||||
if(o.build_area){
|
||||
def newVal = FiledStrand.processArea(o.build_area)
|
||||
if(newVal && newVal.toString()!=o.build_area){
|
||||
println o._id.toString()+" old:"+o.build_area +" new:"+newVal
|
||||
udata.put("build_area", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
if(o.all_area){
|
||||
def newVal = FiledStrand.processArea(o.all_area)
|
||||
if(newVal.toString()!=o.all_area){
|
||||
println o._id.toString()+" old:"+o.all_area +" new:"+newVal
|
||||
udata.put("all_area", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
if(o.standard_area){
|
||||
def newVal = FiledStrand.processStandardArea(o.standard_area)
|
||||
if(newVal.toString()!=o.standard_area){
|
||||
println o._id.toString()+" old:"+o.standard_area +" new:"+newVal
|
||||
udata.put("standard_area", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(o.standard_height){
|
||||
def newVal = FiledStrand.processStandardHeight(o.standard_height)
|
||||
if(newVal.toString()!=o.standard_height){
|
||||
println o._id.toString()+" old:"+o.standard_height +" new:"+newVal
|
||||
udata.put("standard_height", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
if(o.clear_height){
|
||||
def newVal = FiledStrand.processStandardHeight(o.clear_height)
|
||||
if(newVal.toString()!=o.clear_height){
|
||||
println o._id.toString()+" old:"+o.clear_height +" new:"+newVal
|
||||
if(newVal){
|
||||
udata.put("clear_height", newVal.toString())
|
||||
}else{
|
||||
udata.put("clear_height", "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(o.lobby_height){
|
||||
def newVal = FiledStrand.processLobbyHeight(o.lobby_height)
|
||||
if(newVal.toString()!=o.lobby_height){
|
||||
println o._id.toString()+" old:"+o.lobby_height +" new:"+newVal
|
||||
if(newVal){
|
||||
udata.put("lobby_height", newVal.toString())
|
||||
}else{
|
||||
udata.put("lobby_height", "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(o.cubage_percent){
|
||||
def newVal = FiledStrand.processCubagePercent(o.cubage_percent)
|
||||
if(newVal && newVal.toString()!=o.cubage_percent){
|
||||
println o._id.toString()+" old:"+o.cubage_percent +" new:"+newVal
|
||||
udata.put("cubage_percent", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
if(o.virescence_percent){
|
||||
def newVal = FiledStrand.processVirescencePercent(o.virescence_percent)
|
||||
if(newVal.toString()!=o.virescence_percent){
|
||||
println o._id.toString()+" old:"+o.virescence_percent +" new:"+newVal
|
||||
udata.put("virescence_percent", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
if(o.elevators_count){
|
||||
def newVal1 = FiledStrand.processElevators(o.elevators_count)
|
||||
if(newVal1.toString()!=o.elevators_count){
|
||||
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal1
|
||||
udata.put("elevators_count", newVal1.toString())
|
||||
}
|
||||
|
||||
def newVal2 = null
|
||||
if(o.elevators_count.contains("客梯")){
|
||||
newVal2 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("客梯")))
|
||||
if(newVal2.toString()!=o.elevators_count){
|
||||
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal2
|
||||
udata.put("person_elevators_count", newVal2.toString())
|
||||
}
|
||||
}
|
||||
|
||||
def newVal3 = null
|
||||
if(o.elevators_count.contains("货梯")){
|
||||
newVal3 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("货梯")))
|
||||
if(newVal3.toString()!=o.elevators_count){
|
||||
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal3
|
||||
udata.put("goods_elevator_count", newVal3.toString())
|
||||
}
|
||||
}
|
||||
if(newVal2 && newVal3){
|
||||
def total = newVal2.toInteger() + newVal3.toInteger()
|
||||
udata.put("elevators_count", total.toString())
|
||||
println o._id.toString()+" 总电梯:"+total
|
||||
}
|
||||
}
|
||||
|
||||
if(o.goods_elevator_count){
|
||||
def newVal = FiledStrand.processElevators(o.goods_elevator_count)
|
||||
if(newVal.toString()!=o.goods_elevator_count){
|
||||
println o._id.toString()+" old:"+o.goods_elevator_count +" new:"+newVal
|
||||
udata.put("goods_elevator_count", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
if(o.person_elevators_count){
|
||||
def newVal = FiledStrand.processElevators(o.person_elevators_count)
|
||||
if(newVal.toString()!=o.person_elevators_count){
|
||||
println o._id.toString()+" old:"+o.person_elevators_count +" new:"+newVal
|
||||
if(newVal){
|
||||
udata.put("person_elevators_count", newVal.toString())
|
||||
}else{
|
||||
udata.put("person_elevators_count", "")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(o.land_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.land_car_site)
|
||||
if(newVal.toString()!=o.land_car_site){
|
||||
println o._id.toString()+" old:"+o.land_car_site +" new:"+newVal
|
||||
if(newVal){
|
||||
udata.put("land_car_site", newVal.toString())
|
||||
}else{
|
||||
udata.put("land_car_site", "")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
if(o.underground_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.underground_car_site)
|
||||
if(newVal.toString()!=o.underground_car_site){
|
||||
println o._id.toString()+" old:"+o.underground_car_site +" new:"+newVal
|
||||
if(newVal){
|
||||
udata.put("underground_car_site", newVal.toString())
|
||||
}else{
|
||||
udata.put("underground_car_site", "")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(o.is_register){
|
||||
def newVal = FiledStrand.processRegister(o.is_register)
|
||||
if(newVal.toString()!=o.is_register){
|
||||
println o._id.toString()+" old:"+o.is_register +" new:"+newVal
|
||||
udata.put("is_register", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(o.property_life){
|
||||
def newVal = FiledStrand.processPropertyLife(o.property_life)
|
||||
udata.put("property_life", newVal)
|
||||
@@ -173,7 +336,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
if(o.finish_building_date){
|
||||
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
|
||||
udata.put("finish_building_date", newVal.toString())
|
||||
if(newVal && newVal.toString()!=o.finish_building_date){
|
||||
println o._id.toString()+" old:"+o.finish_building_date +" new:"+newVal
|
||||
udata.put("finish_building_date", newVal.toString())
|
||||
}
|
||||
}
|
||||
|
||||
if(o.real_use_percent){
|
||||
@@ -181,21 +347,14 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
udata.put("real_use_percent", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.underground_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.underground_car_site)
|
||||
udata.put("underground_car_site", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.land_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.land_car_site)
|
||||
udata.put("land_car_site", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.parking_fee){
|
||||
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
|
||||
udata.put("parking_fee", newVal)
|
||||
}
|
||||
|
||||
def addr_alias = o.addr_alias
|
||||
if(addr_alias){
|
||||
def addrs = []
|
||||
for(def arr : addr_alias){
|
||||
@@ -214,13 +373,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
// if(o.estate_charge && !o.estate_charge.isNumber()){
|
||||
//// def val = o.estate_charge.toString().toDouble()
|
||||
//// if(val>60 || val < 0.35){
|
||||
//// println val
|
||||
// udata.put("estate_charge", null)
|
||||
//// }
|
||||
// }
|
||||
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
def city = "³£ÖÝ"
|
||||
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 根据链家数据修改小区库中字段
|
||||
* 使用到中间映射表
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class UpdateDistrictField extends BaseBatch{
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
|
||||
|
||||
def sources = ["链家", "我爱我家", "安居客"]
|
||||
|
||||
def statMap = [:]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new UpdateDistrictField(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def sourceMap = [:]
|
||||
sources.each {source->
|
||||
def valMap = findThirdMatchDist(source)
|
||||
println source +" "+valMap.size()
|
||||
if(valMap){
|
||||
sourceMap.put(source, valMap)
|
||||
}
|
||||
}
|
||||
def margeResults = megerDatas(sourceMap)
|
||||
updateDB(margeResults)
|
||||
println "修改完成 "+new Date().toString()
|
||||
}
|
||||
|
||||
def megerDatas(sourceMap){
|
||||
def set = new HashSet()
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
set.addAll(valMap.keySet())
|
||||
}
|
||||
def lastValues = []
|
||||
set.each {id->
|
||||
|
||||
def dbObj = CommonDatautils.findDistrictById(id)
|
||||
|
||||
def datas = []
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
def sourceEsObj = findCommunities(valMap.get(id))
|
||||
if(sourceEsObj){
|
||||
if(source == "链家"){
|
||||
sourceEsObj.source = "lianjia"
|
||||
}else if(source == "我爱我家"){
|
||||
sourceEsObj.source = "wiwj"
|
||||
}else if(source == "安居客"){
|
||||
sourceEsObj.source = "anjuke"
|
||||
}
|
||||
datas << sourceEsObj
|
||||
}
|
||||
}
|
||||
if(dbObj && datas){
|
||||
def newValMap = getNewField2(datas, dbObj)
|
||||
if(newValMap){
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
statMap.each {source,fieldMap->
|
||||
println source +" "+fieldMap.size()
|
||||
fieldMap.each{f,v->
|
||||
println f+" "+ v
|
||||
}
|
||||
println "----------------------------------------"
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
|
||||
|
||||
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||
// def lianjisIds = lianjiaMap.keySet()
|
||||
// def wiwjIds = wiwjMap.keySet()
|
||||
// def anjukeIds = anjukeMap.keySet()
|
||||
// def set = new HashSet()
|
||||
// set.addAll(lianjisIds)
|
||||
// set.addAll(wiwjIds)
|
||||
// set.addAll(anjukeIds)
|
||||
//
|
||||
// def lastValues = []
|
||||
// def lianjiaUpdateCount = 0
|
||||
// def wiwjUpdateCount = 0
|
||||
// def anjukeUpdateCount = 0
|
||||
// def lianjiaCountMap = [:]
|
||||
// def wiwjCountMap = [:]
|
||||
// def anjukeCountMap = [:]
|
||||
// set.each {id->
|
||||
// BasicDBObject newValMap = new BasicDBObject()
|
||||
// fields.each {field->
|
||||
// def lianjiaValMap = lianjiaMap.get(id)
|
||||
// def wiwjValMap = wiwjMap.get(id)
|
||||
// def anjukeValMap = anjukeMap.get(id)
|
||||
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||
// newValMap.put(field, lianjiaValMap.get(field))
|
||||
// lianjiaUpdateCount++
|
||||
// def count = lianjiaCountMap.get(field)
|
||||
// if(!count){
|
||||
// lianjiaCountMap.put(field, 1)
|
||||
// }else{
|
||||
// lianjiaCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||
// newValMap.put(field, wiwjValMap.get(field))
|
||||
// wiwjUpdateCount++
|
||||
// def count = wiwjCountMap.get(field)
|
||||
// if(!count){
|
||||
// wiwjCountMap.put(field, 1)
|
||||
// }else{
|
||||
// wiwjCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||
// newValMap.put(field, anjukeValMap.get(field))
|
||||
// anjukeUpdateCount++
|
||||
// def count = anjukeCountMap.get(field)
|
||||
// if(!count){
|
||||
// anjukeCountMap.put(field, 1)
|
||||
// }else{
|
||||
// anjukeCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// newValMap.put("id", id)
|
||||
// lastValues << newValMap
|
||||
// }
|
||||
//
|
||||
// lianjiaCountMap.each {k,v->
|
||||
// println "[链家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// wiwjCountMap.each {k,v->
|
||||
// println "[我爱我家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// anjukeCountMap.each {k,v->
|
||||
// println "[安居客] "+k+" "+v
|
||||
// }
|
||||
// return lastValues
|
||||
// }
|
||||
|
||||
def updateDB(margeResults){
|
||||
for(def mr : margeResults){
|
||||
def id = mr.get("id")
|
||||
mr.remove("id")
|
||||
mr.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
|
||||
}
|
||||
}
|
||||
|
||||
// def batchUpdateDist(source){
|
||||
//// def rets = findThirdMatchDist(source)
|
||||
// println "[${source}] 符保条件小区量:"+rets.size()
|
||||
// def updateIndex = new AtomicLong()
|
||||
// final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
// def allUpdateList = new CopyOnWriteArrayList()
|
||||
//
|
||||
// def sourceMap = new ConcurrentHashMap()
|
||||
//
|
||||
// for(def i=0;i<rets.size();i++) {
|
||||
// final def ret = rets.get(i)
|
||||
//// fixedThreadPool.execute(
|
||||
//// new Runnable() {
|
||||
//// void run() {
|
||||
// try {
|
||||
// def newValMap = running(ret.url, ret.old_match_id)
|
||||
// if(newValMap){
|
||||
// sourceMap.put(ret.old_match_id, newValMap)
|
||||
// allUpdateList << newValMap
|
||||
// updateIndex.addAndGet(1)
|
||||
// }
|
||||
// }catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
//// }
|
||||
//// }
|
||||
//// )
|
||||
// }
|
||||
// latch.await()
|
||||
// println "总修改量:"+updateIndex.get()
|
||||
// def countMap = [:]
|
||||
// for(def map : allUpdateList){
|
||||
// map.each{k,v->
|
||||
// def count = countMap.get(k)
|
||||
// if(!count){
|
||||
// countMap.put(k, 1)
|
||||
// }else{
|
||||
// countMap.put(k, count+1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// countMap.each {k,v->
|
||||
// println k +" 更新量:"+v
|
||||
// }
|
||||
//
|
||||
// println "---------------------------------------------------------"
|
||||
// return sourceMap
|
||||
// }
|
||||
|
||||
def running(url, old_match_id){
|
||||
def esObj = findCommunities(url)
|
||||
def dbObj = CommonDatautils.findDistrictById(old_match_id)
|
||||
if(dbObj){
|
||||
def newValMap = getNewField(esObj, dbObj)
|
||||
return newValMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getNewField(esObj, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
esObj.each{field, esVal->
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(esVal && field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
|
||||
}
|
||||
if(esVal && field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(esVal && field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
newValMap.put(field, newVal)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return newValMap
|
||||
}
|
||||
|
||||
def getNewField2(esDatas, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
def lastValueMap = [:]
|
||||
def haveValueMap = [:]
|
||||
fields.each{field->
|
||||
for(def esData : esDatas){
|
||||
def esVal = esData.get(field)
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(haveValueMap.containsKey(field)){
|
||||
continue
|
||||
}else{
|
||||
if(lastValueMap.containsKey(field)){
|
||||
continue
|
||||
}
|
||||
if(esVal){
|
||||
|
||||
if(field=="estate_charge" || field=="car_site_count") {
|
||||
esVal = FiledStrand.getNumber(esVal)
|
||||
}
|
||||
if(dbVal && esVal.toString()==dbVal.toString()){
|
||||
haveValueMap.put(field, esVal)
|
||||
}else{
|
||||
if(field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
lastValueMap.put(field, newVal)
|
||||
|
||||
//以下为统计各网站各字段的更新情况
|
||||
def fieldStatMap = statMap.get(esData.source)
|
||||
if(!fieldStatMap){
|
||||
fieldStatMap = [:]
|
||||
statMap.put(esData.source, fieldStatMap)
|
||||
}
|
||||
def count = fieldStatMap.get(field)
|
||||
if(!count){
|
||||
fieldStatMap.put(field, 1)
|
||||
}else{
|
||||
fieldStatMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValueMap
|
||||
}
|
||||
|
||||
def findThirdMatchDist(source){
|
||||
def query = new BasicDBObject("source",source).append("old_match_way", 1)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("url", 1)
|
||||
keys.put("old_match_id", 1)
|
||||
keys.put("old_match_city", 1)
|
||||
keys.put("old_match_district", 1)
|
||||
keys.put("old_match_name", 1)
|
||||
def idsMap = [:]
|
||||
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
|
||||
for(def db : dbcursor){
|
||||
if(!db.old_match_id && db.old_match_name){
|
||||
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
|
||||
if(dbo){
|
||||
idsMap.put(dbo._id.toString(), db.url)
|
||||
}
|
||||
}else{
|
||||
idsMap.put(db.old_match_id, db.url)
|
||||
}
|
||||
|
||||
}
|
||||
return idsMap
|
||||
}
|
||||
|
||||
static def findCommunities(url){
|
||||
if(!url){
|
||||
return null
|
||||
}
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||
def rets = []
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
}catch(Exception e){
|
||||
findCommunities(url)
|
||||
}
|
||||
def data = [:]
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
data.put("household_count", d.households)
|
||||
data.put("loudong_count", d.loudong_cnt)
|
||||
data.put("developer", d.developer)
|
||||
data.put("estate_company", d.property_company)
|
||||
data.put("estate_charge", d.property_fee)
|
||||
data.put("car_site_count", d.parking_cnt)
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
+76
-2
@@ -14,7 +14,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class EsMonthHouseStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
@@ -26,7 +26,8 @@ class EsMonthHouseStat {
|
||||
// task.statHouseUpdateCount()
|
||||
|
||||
// task.statDistHouse()
|
||||
task.statRent()
|
||||
// task.statRent()
|
||||
// task.statOfficeHouse()
|
||||
}
|
||||
|
||||
def statRent(){
|
||||
@@ -201,6 +202,51 @@ class EsMonthHouseStat {
|
||||
},0)
|
||||
}
|
||||
|
||||
def statOfficeHouse(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allLists = []
|
||||
allLists << ["城市", "近一年写字楼出售量", "写字楼量", "近一年出售房源量大于3的写字楼量", "大于3写字楼量占比"]
|
||||
citys.each{c->
|
||||
def count = findHouseCountByCat(c, "office_sell",20240101, 20250201)
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = 0
|
||||
if(count){
|
||||
distCount = findHouseCountByCat(dist.city, dist.region, alias, 20240101, 20250201, "office_sell")
|
||||
if(distCount && distCount>=3){
|
||||
disthouseCount++
|
||||
}
|
||||
}
|
||||
// println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
// println districts.size()
|
||||
def precent = 0
|
||||
if(districts){
|
||||
precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
}
|
||||
def row = [c , count, districts.size(), disthouseCount, precent+"%"]
|
||||
println row
|
||||
allLists << row
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\全国写字楼情况占比统计.xlsx", allLists, "写字楼情况")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
@@ -219,6 +265,23 @@ class EsMonthHouseStat {
|
||||
return total
|
||||
}
|
||||
|
||||
def findHouseCountByCat(def city, def region, def alias, def startDate, def endDate, cat){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
@@ -240,6 +303,17 @@ class EsMonthHouseStat {
|
||||
}
|
||||
|
||||
|
||||
def findHouseCountByCat(def city, def cat, def startDate, def endDate){
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user