Merge remote-tracking branch 'origin/master'

This commit is contained in:
2025-04-16 18:32:36 +08:00
306 changed files with 20333 additions and 3427 deletions
+18
View File
@@ -537,6 +537,24 @@
<version>1.70</version> <!-- 确保使用最新版本 -->
</dependency>
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>1.18.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.hankcs.hanlp.restful</groupId>
<artifactId>hanlp-restful</artifactId>
<version>0.0.12</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -1,6 +1,6 @@
package com.fangjia.script
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.*
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
@@ -112,6 +112,18 @@ abstract class BaseBatch {
}
if(SystemUtils.IS_OS_LINUX){
sendDingDingMsg("2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c",msg)
try {
Map<String, String> params = new HashMap<String, String>();
params.put("phone", "13764369683");
params.put("type", "taskFail");
params.put("msg", batchName.substring(batchName.length() - 20, batchName.length()));
HttpRequest httpRequest = new HttpRequest();
httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8");
} catch (Exception ex) {
}
task.exception = msg.toString()
}
}
@@ -21,7 +21,7 @@ import java.sql.DriverManager
@BatchOwner(Watcher.WangWenYan)
class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo", 27017)
static void main(String[] args) {
runBatch(new ClickHouseDistrictStatMarkupCalcBatch(), args)
}
@@ -35,7 +35,7 @@ class DatabaseStatWatcherBatch extends BaseBatch {
def yesterday = DateUtil.add(today, -1)
for(def host : hosts){
Mongo mongo = new Mongo(new MongoURI("mongodb://${host}"))
Mongo mongo = new Mongo(new MongoURI("mongodb://developer:G00IDCDEV12385@${host}:28017"))
for(def databaseName : mongo.getDatabaseNames()){
if(NOT_WATCH_DBS.contains(databaseName)){
continue
@@ -18,7 +18,7 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.WangWenYan)
class DistrictStatMarkupCalcBatch extends BaseBatch {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 27017)
Date oneYearAgo = DateUtil.add(new Date(), Calendar.MONTH, -1)
@@ -1,16 +1,17 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
//没有成交时间,随机生成
@@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicLong
*/
class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
@@ -43,7 +43,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -71,8 +71,7 @@ class BigdataDealBatch extends BaseBatch{
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -61,25 +61,25 @@ class CcbCommonUtils {
def countStr = args[0]
if(countStr=="344"){
citys = CcbCommonUtils.citys_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="92"){
citys = CcbCommonUtils.citys_92
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="252"){
citys = CcbCommonUtils.citys_252
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="all"){
citys = CcbCommonUtils.citys_fjw_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
}else{
citys = CcbCommonUtils.citys_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
println "运行城市数量:"+citys.size()
return citys;
@@ -45,7 +45,7 @@ class DistrictDelete {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictDelete()
@@ -1353,13 +1353,14 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\更新人工指导价_0314.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['J'])
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['C'] = dist.city
d['D'] = dist.region
d['E'] = dist.name
d['F'] = dist.status
}
})
@@ -27,11 +27,14 @@ class ThirdPartyPriceBatch extends BaseBatch {
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
"淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12","房多多":"13","我爱我家":"14",
"禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20",
"乐有家":"21","幸福里":"22","中国房价行情":"23","常州住房":"24"
]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -263,6 +266,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(third.get("csjj_esf_price")){
price_map.csjj_avg_price = third.get("csjj_esf_price")
}
if(third.get("leyoujia_esf_price")){
price_map.leyoujia_avg_price = third.get("leyoujia_esf_price")
}
if(third.get("c21_esf_price")){
price_map.c21_avg_price = third.get("c21_esf_price")
}
if(third.get("jiwu_esf_price")){
price_map.jiwu_avg_price = third.get("jiwu_esf_price")
}
if(third.get("xingfuli_esf_price")){
price_map.xingfuli_avg_price = third.get("xingfuli_esf_price")
}
if(third.get("zwfc_esf_price")){
price_map.zwfc_avg_price = third.get("zwfc_esf_price")
}
if(third.get("creprice_esf_price")){
price_map.creprice_avg_price = third.get("creprice_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -313,6 +340,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
web = "房产超市"
}else if(k=="csjj_avg_price"){
web = "城市居家"
}else if(k=="leyoujia_avg_price"){
web = "乐有家"
}else if(k=="c21_avg_price"){
web = "C21"
}else if(k=="jiwu_avg_price"){
web = "吉屋网"
}else if(k=="xingfuli_avg_price"){
web = "幸福里"
}else if(k=="creprice_avg_price"){
web = "中国房价行情"
}else if(k=="zwfc_avg_price"){
web = "常州住房"
}
ret << sourceMap.get(web)
@@ -603,6 +642,30 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
}
if(!dataMap.get("leyoujia_esf_price") && db.get("leyoujia_esf_price") && db.get("leyoujia_esf_price")>3000){
dataMap.put("leyoujia_esf_price", db.get("leyoujia_esf_price"))
}
if(!dataMap.get("c21_esf_price") && db.get("c21_esf_price") && db.get("c21_esf_price")>3000){
dataMap.put("c21_esf_price", db.get("c21_esf_price"))
}
if(!dataMap.get("jiwu_esf_price") && db.get("jiwu_esf_price") && db.get("jiwu_esf_price")>3000){
dataMap.put("jiwu_esf_price", db.get("jiwu_esf_price"))
}
if(!dataMap.get("xingfuli_esf_price") && db.get("xingfuli_esf_price") && db.get("xingfuli_esf_price")>3000){
dataMap.put("xingfuli_esf_price", db.get("xingfuli_esf_price"))
}
if(!dataMap.get("creprice_esf_price") && db.get("creprice_esf_price") && db.get("creprice_esf_price")>3000){
dataMap.put("creprice_esf_price", db.get("creprice_esf_price"))
}
if(!dataMap.get("zwfc_esf_price") && db.get("zwfc_esf_price") && db.get("zwfc_esf_price")>3000){
dataMap.put("zwfc_esf_price", db.get("zwfc_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -36,7 +36,7 @@ class TradeStatLog {
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -18,10 +18,10 @@ import com.mongodb.QueryOperators
import groovy.sql.Sql
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "mongo.db.fangjia.com")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -12,7 +12,7 @@ class MatchDealPriceRegion {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -21,7 +21,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
*/
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -18,7 +18,7 @@ class FumingTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -38,7 +38,7 @@ class HailanTest {
}
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
@@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong
class PocStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
@@ -15,7 +15,7 @@ class RuiFenPocTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -7,6 +7,7 @@ import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.newDistrict.InvokeHttpRequest
import com.fangjia.script.util.*
@@ -98,11 +99,13 @@ class CommonDatautils {
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com", 28017)
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -259,7 +262,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","托斯卡纳庄园")
// q.put("name","新泾三村")
DBObject keys = new BasicDBObject();
if(fields){
@@ -597,9 +600,9 @@ class CommonDatautils {
keys.put("region", 1);
keys.put("name", 1);
keys.put("_id", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
Iterator<DBObject> iters = cursor.iterator()
def result = [];
def result = new HashSet();
try{
while(iters.hasNext()) {
DBObject obj = iters.next()
@@ -755,7 +758,10 @@ class CommonDatautils {
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("region", region);
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
@@ -786,7 +792,8 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
@@ -814,7 +821,9 @@ class CommonDatautils {
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
if(!d.township){
rets << d
}
}
return rets;
}
@@ -1131,6 +1140,26 @@ class CommonDatautils {
return ret;
}
public static def findNormalDistrict(String city, String region, String name, String cat, List fields){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
if(region){
q.put("region", region);
}
q.put("status",0)
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
return SEAWEED_COLL.findOne(q, keys);
}
public static def findDistrictsByCityAndName(String city, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
@@ -4209,19 +4238,21 @@ class CommonDatautils {
static def findAllNewPoiDatas(city){
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
// if(map.address && map.address.contains("五角场")){
index++
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
// }
})
return rets;
}
static def findOneThirdNetWorkOffice(city, region, name){
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
def map = [:]
@@ -4234,6 +4265,16 @@ class CommonDatautils {
return map
}
static def findThirdNetWorkOffices(city, region, name){
String sql = "select * from office where city='${city}' and region='${region}' and name='${name}'"
def datas = []
ElasticUtils.executeSqlQuery(sql, { m->
datas << m
})
return datas
}
static def findThirdNetWorkOffice(){
String sql = "select city,region,name,address,location from office where city='烟台' or city='北京'"
@@ -4678,6 +4719,14 @@ class CommonDatautils {
}
static def httpSaveData89(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString())
doPost(newUrl,header, params, true, "utf-8")
}
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
@@ -4965,6 +5014,20 @@ class CommonDatautils {
return regionMap
}
static def queryRangeByCat(cat){
def q = new BasicDBObject()
q.put("type",cat)
q.put("shape",new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("city_short",1)
keys.put("district_short",1)
keys.put("city",1)
keys.put("district",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList()
}
static def queryAllRegionNameByCity(){
@@ -5135,21 +5198,66 @@ class CommonDatautils {
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def distName = null
if(d.mzb_district_name){
def list = townMap.get(d.mzb_district_name)
distName = d.mzb_district_name
}else{
distName = d.district
}
if(distName){
def list = townMap.get(distName)
if(!list){
list = []
}
list << d
townMap.put(d.mzb_district_name, list)
townMap.put(distName, list)
}
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryAllTown(fields){
def q = new BasicDBObject()
q.put("type","乡镇街道")
def keys = new BasicDBObject()
for(def key : fields){
keys.put(key,1)
}
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def cityMap = [:]
for(def d : dbcursor){
def townMap = cityMap.get(d.city)
if(!townMap){
townMap = [:]
cityMap.put(d.city, townMap)
}
def distName = null
if(d.mzb_district_name){
distName = d.mzb_district_name
}else{
distName = d.district
}
if(distName){
def list = townMap.get(distName)
if(!list){
list = []
}
list << d
townMap.put(distName, list)
}
}
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return cityMap
}
static def queryFullRegionTownByCity(city, fields){
def q = new BasicDBObject()
@@ -5515,4 +5623,25 @@ class CommonDatautils {
return 0
}
static def findBusinessByCity(city){
def q = new BasicDBObject()
q.put("short_city", city)
return bk_business_coll.find(q).toList()
}
static def findOfficeByMap(city, small_type){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("small_type", small_type))
def rets = []
def index= 0
ElasticUtils89.executeQuery("192.168.0.89", "gaode_map_poi_2024", queryBuilder, { String id, Map map->
// if(map.address && map.address.contains("五角场")){
index++
rets << ["map_id":map.data_id,"province":map.province,"city":map.city, "district":map.district, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.lon, "lat":map.location.lat, "parent_id":map.parent]
// }
})
return rets;
}
}
@@ -23,7 +23,7 @@ import org.apache.commons.lang.StringUtils
class FewerHouseDistrictCrawlCommandBatch extends BaseBatch {
private static
final DBCollection ARTEMIS_COLL = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com")
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static void main(String[] args) {
runBatch(new FewerHouseDistrictCrawlCommandBatch(), args)
@@ -15,7 +15,7 @@ import org.bson.types.ObjectId
//new FewerHouseDistrictCrawlCommandBatch().process(null);
//return;
//
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "192.168.0.43", 27017);
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com", 27017);
//DBCursor cursor_artemis_message = col_43_artemis_message.find(new BasicDBObject("status", 0));
//int count = 0;
//while(cursor_artemis_message.hasNext()){
@@ -1,8 +1,18 @@
package com.fangjia.script.dataAna
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.*
import org.apache.commons.codec.digest.DigestUtils
import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch
@@ -10,30 +20,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Semaphore
import org.apache.commons.codec.digest.DigestUtils
import org.bson.types.ObjectId
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.google.common.collect.Lists
import com.google.common.collect.Sets
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
@BatchOwner(Watcher.LiXue)
class CityInfoAnaBatch extends BaseBatch {
@@ -53,8 +39,10 @@ class CityInfoAnaBatch extends BaseBatch {
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
final static DBCollection OFFICE_RENT_STAT_CONN = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", MongoUtils.MongoDB_Domain_mongo);
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
final static DSCollection SEAWEED = DbUtils.seaweed
// final static DSCollection SEAWEED = DbUtils.seaweed
static final DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final static def CITYS = [];
@@ -125,7 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v ->
// if(["南京"].contains(k.toString())){
// if(["北京"].contains(k.toString())){
CITYS << k.toString();
// }
}
@@ -140,72 +128,47 @@ class CityInfoAnaBatch extends BaseBatch {
// 城市小区数
def allCityDisCount = [:];
LinkedHashMap<String, Object> facet = new LinkedHashMap<String, Object>(2);
facet.put("fields", ["city"]);
facet.put('limit', 500);
DBObject disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'district');
DSCursor sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
List<FacetResult> disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityDisCount.put(it.name, it.count.intValue());
}
BasicDBObject match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "district"));
DBObject groupFields = new BasicDBObject();
groupFields.put("_id", new BasicDBObject("city", '$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
AggregationOutput output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityDisCount.put(obj.get("city"), result.get("count"))
}
// 城市新盘数
def allCityNewDisCount = [:];
disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'new_district');
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityNewDisCount.put(it.name, it.count.intValue());
}
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "new_district"));
output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityNewDisCount.put(obj.get("city"), result.get("count"))
}
// 城市写字楼数
def allCityOfficeCount = [:];
disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'office');
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "office"));
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityOfficeCount.put(it.name, it.count.intValue());
}
output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityOfficeCount.put(obj.get("city"), result.get("count"))
}
// 城市商铺数
def allCityShopCount = [:];
disQuery = new BasicDBObject();
disQuery.put('$facet', facet);
disQuery.put("visible", 0);
disQuery.put("cat", 'shop');
match = new BasicDBObject('$match', new BasicDBObject("status", 0).append("cat", "shop"));
sellDSCursor = SEAWEED.find(disQuery).skip(0).limit(0);
disFacetResult = sellDSCursor.getFacetResults();
if (disFacetResult) {
def obj = disFacetResult[0].values;
obj.each {
allCityShopCount.put(it.name, it.count.intValue());
}
output = SEAWEED.aggregate(Arrays.asList(match, group));
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
allCityShopCount.put(obj.get("city"), result.get("count"))
}
// 城市均价
def allCityStats = [:];
query = new BasicDBObject();
@@ -220,10 +183,10 @@ class CityInfoAnaBatch extends BaseBatch {
doProcess(allCitys.get(it), weekend, laseWeekend, allHouseStats, allCityDisCount, allCityNewDisCount, allCityOfficeCount, allCityShopCount, allCityStats);
}
//统计有楼栋的小区数和有房号的小区数
// countDistrictsHasLoudong(allCitys, weekend)
//countDistrictsHasLoudongNew(allCitys, weekend)
// //统计有楼栋的小区数和有房号的小区数
//// countDistrictsHasLoudong(allCitys, weekend)
// //countDistrictsHasLoudongNew(allCitys, weekend)
//
final CountDownLatch latch = new CountDownLatch(CITYS.size())
final Semaphore semaphore = new Semaphore(10)
CITYS.each {
@@ -239,13 +202,13 @@ class CityInfoAnaBatch extends BaseBatch {
} finally {
semaphore.release()
latch.countDown()
}
}
};
})
}
latch.await()
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t楼栋覆盖率统计完成"
latch = new CountDownLatch(CITYS.size())
semaphore = new Semaphore(10)
CITYS.each {
@@ -267,12 +230,12 @@ class CityInfoAnaBatch extends BaseBatch {
}
latch.await()
println "${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())}\t房号覆盖率统计完成"
//71个城市的覆盖率统计
sendCountData(weekend)
//最后统计之前因为太慢取消统计的本周估价次数
latch = new CountDownLatch(CITYS.size())
semaphore = new Semaphore(10)
@@ -292,14 +255,14 @@ class CityInfoAnaBatch extends BaseBatch {
query.put('city', cityName);
query.put('c_date', new BasicDBObject('$gte', laseWeekend).append('$lte', weekend));
entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));
if (entity.get('evaluation_count')) {
entity.put('price_to_eva', DataUtil.roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
}
CITY_INFO_CONN.save(entity)
println cityName+"本周估价次数统计完毕"
}
} catch (Exception e) {
println e.getMessage()
} finally {
@@ -343,21 +306,21 @@ class CityInfoAnaBatch extends BaseBatch {
def totalDbhouseloudongCount = HOUSE_LOUDONG_CONN.count(findObj)
datas << ['', totalDistrictCount, totalLoudongCount, df.format(totalLoudongCount * 1.0/ totalDistrictCount), totalFanghaoCount, df.format(totalFanghaoCount * 1.0/ totalDistrictCount), totalDbLoudongCount, totalDbhouseloudongCount]
ExcelUtils.write(fileName, datas, resultTitle)
String id = ObjectId.get().toString()
BasicDBObject file = new BasicDBObject()
file.put("t", "upload")
file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
file.put("c_date", new Date())
file.put("_id", idGenerator(id, "o"))
file.put("data", new File(fileName).getBytes())
fsDB.save(file)
def href = DataUtil.encodePicUrl(id, "o", "upload")
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
receivers.each {receiver->
jms.sendMail(receiver, resultTitle, href)
}
new File(fileName).deleteOnExit()
// ExcelUtils.write(fileName, datas, resultTitle)
// String id = ObjectId.get().toString()
// BasicDBObject file = new BasicDBObject()
// file.put("t", "upload")
// file.put("content_type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
// file.put("c_date", new Date())
// file.put("_id", idGenerator(id, "o"))
// file.put("data", new File(fileName).getBytes())
// fsDB.save(file)
// def href = DataUtil.encodePicUrl(id, "o", "upload")
// href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
// receivers.each {receiver->
// jms.sendMail(receiver, resultTitle, href)
// }
// new File(fileName).deleteOnExit()
}
private String idGenerator(String url, String size) {
@@ -585,8 +548,8 @@ class CityInfoAnaBatch extends BaseBatch {
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 小区完成]"
// if (allCityNewDisCount.get(cityName)) {
// def newDisCount = allCityNewDisCount.get(cityName);
// if (allCityNewDisCount.get(cityName)) {
// def newDisCount = allCityNewDisCount.get(cityName);
// entity.put('new_district_detail', newDistrictDetail(newDisCount, cityName));
// } else {
// entity.put('new_district_detail', [:]);
@@ -604,23 +567,30 @@ class CityInfoAnaBatch extends BaseBatch {
if (officePrices) {
detail.put('price', roundDouble(officePrices.size(), officeCount));
}
query = new BasicDBObject();
query.put('city', cityName);
query.put('kind', '写字楼');
query.put('weekend', weekend);
def officeRents = RENT_STAT_CONN.distinct("name", query);
def officeRents = findOfficeRent(cityName)
if (officeRents) {
detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
}
def halfYearofficeRentCount = findHalfYearOfficeRentCount(cityName)
if (halfYearofficeRentCount) {
detail.put('halfYearRentPrice', roundDouble(halfYearofficeRentCount, officeCount));
}
entity.put('office_detail', detail);
entity.put('existing_office_count', officeCount);
} else {
entity.put('office_detail', [:]);
}
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
// if(curInfo){
// CITY_INFO_CONN.update(new BasicDBObject("_id", curInfo._id), new BasicDBObject('$set', entity))
// print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 写字楼完成]"
// }
if (entity.get('official_population') && entity.get('district_avg_population')) {
entity.put('prediction_district_count', (int) (entity.get('official_population') / entity.get('district_avg_population')));
@@ -628,12 +598,12 @@ class CityInfoAnaBatch extends BaseBatch {
entity.put('district_coverage', roundDouble(entity.get('existing_district_count'), entity.get('prediction_district_count')));
}
}
if (allCityShopCount.get(cityName)) {
def shopCount = allCityShopCount.get(cityName);
def detail = shopDetail(shopCount, cityName);
DBObject query = new BasicDBObject();
query.put('city', cityName);
//query.put('estate_type', '商铺');
@@ -642,7 +612,7 @@ class CityInfoAnaBatch extends BaseBatch {
if (shopPrices) {
detail.put('price', roundDouble(shopPrices.size(), shopCount));
}
query = new BasicDBObject();
query.put('city', cityName);
query.put('kind', '商铺');
@@ -651,45 +621,41 @@ class CityInfoAnaBatch extends BaseBatch {
if (shopRents) {
detail.put('rentPrice', roundDouble(shopRents.size(), shopCount));
}
entity.put('shop_detail', detail);
entity.put('existing_shop_count', shopCount);
}else {
entity.put('shop_detail', [:]);
}
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 商铺完成]"
def cityStat = allCityStats.get(cityName);
if (cityStat) {
entity.put('avg_price', cityStat.avg_price);
}
DBObject query = new BasicDBObject();
query.put('city', cityName);
query.put('c_date', new BasicDBObject('$gte', lastWeekend).append('$lte', weekend));
//entity.put('evaluation_count', (int) EVALUATION_CONN.count(query));由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
entity.put('price_total_count', (int) PRICE_JIUCUO_CONN.count(query));
query.put('price_lev', '偏高');
entity.put('price_high_count', (int) PRICE_JIUCUO_CONN.count(query));
query.put('price_lev', '偏低');
entity.put('price_low_count', (int) PRICE_JIUCUO_CONN.count(query));
/*if (entity.get('evaluation_count')) {由于估价记录数据量大,查询慢,先去掉这个,最后面在用多线程统计
entity.put('price_to_eva', roundDouble(entity.get('price_total_count'), entity.get('evaluation_count')));
}*/
if (entity.get('price_high_count')) {
entity.put('low_to_high', roundDouble(entity.get('price_low_count'), entity.get('price_high_count')));
}
print "\t[${DateUtil.formatToTightYYYYMMDDhhmmss(new Date())} 估价完成]"
// query = new BasicDBObject();
// query.put('city', cityName);
// query.put('weekend', weekend);
@@ -710,7 +676,7 @@ class CityInfoAnaBatch extends BaseBatch {
// }
// }
// }
//
//
// if (total > 0) {
// BigDecimal bg = new BigDecimal(webZhunque * 1.0 / total);
// entity.put('web_average', bg.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue());
@@ -1284,7 +1250,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1)
}
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def countMap = [:]
println cur.count()
@@ -1335,7 +1301,7 @@ class CityInfoAnaBatch extends BaseBatch {
fields.put(key, 1)
}
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
def cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def countMap = [:]
@@ -1546,6 +1512,69 @@ class CityInfoAnaBatch extends BaseBatch {
}
return true;
}
def findOfficeRent(cityName){
def query = new BasicDBObject();
query.put('city', cityName);
// query.put('kind', '写字楼');
// query.put('weekend', weekend);
// def officeRents = RENT_STAT_CONN.distinct("name", query);
// if (officeRents) {
// detail.put('rentPrice', roundDouble(officeRents.size(), officeCount));
// }
query.put('cat', 'office');
def date1 = new Date()
date1.setDate(28)
if(new Date().getTime()> date1.getTime()){
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")));
}else{
query.put('date', Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")));
}
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
return officeRents
}
def findHalfYearOfficeRentCount(cityName){
def date1 = new Date()
date1.setDate(28)
def months = []
if(new Date().getTime()> date1.getTime()){
for(def i=1;i<=6;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}else{
for(def i=2;i<=7;i++){
months << Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1*i, "yyyyMM"))
}
}
def map = [:]
for(def month : months){
def query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'office');
query.put('date', month);
def officeRents = OFFICE_RENT_STAT_CONN.distinct("name", query);
officeRents.each{k->
def count = map.get(k)
if(!count){
map.put(k, 1)
}else{
map.put(k, count + 1)
}
}
}
def total = 0
map.each{k,v->
if(v==6){
total ++
}
}
return total
}
}
@@ -21,8 +21,8 @@ import java.util.concurrent.TimeUnit
@BatchOwner(Watcher.GaoJuan)
class DistrictTradeHouseStatBatch extends BaseBatch{
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "192.168.0.43")
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new DistrictTradeHouseStatBatch(), args)
@@ -0,0 +1,140 @@
package com.fangjia.script.dealPrice
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
/**
* 成交同步206,按日同步
*/
@BatchOwner(Watcher.GaoJuan)
class SynchDealPriceToEs extends BaseBatch{
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
static main(args) {
runBatch(new SynchDealPriceToEs(), args)
}
@Override
void process(Object args) {
run();
}
def run(){
def startDate = DateUtils.formatDateByPattern(new Date()-1, "yyyy-MM-dd")
def q = new BasicDBObject()
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcursor = dealPriceConn.find(q)
println DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd")+" 更新量:"+ dbcursor.count()
def ret = []
for(def o : dbcursor){
ret << o
}
save(ret)
// def batchsize = 2000
// def citys = getCityCount()
// citys.each{city,count->
// if(city instanceof String){
// for(def year = 1999;year<=2025;year++){
// def q = new BasicDBObject("city", city)
// q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(year+"-01-01", "yyyy-MM-dd")).append('$lte', DateUtils.parseFromTightByPattern(year+1+"-01-01", "yyyy-MM-dd")))
// def dbcursor = dealPriceConn.find(q).batchSize(batchsize)
// println city +" "+ dbcursor.count()
// def ret = []
// for(def o : dbcursor){
// ret << o
// }
// save(ret)
// }
// }
// }
}
def getCityCount(){
BasicDBObject match = null;
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
groupFields.put("_id", new BasicDBObject("city",'$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city")) {
int count = (Integer) result.get("count");
map.put(obj.get("city"), count)
}
}
return map;
}
def save(rows){
BulkRequest request = new BulkRequest();
int batchSize = 500;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.206")
for(def map : rows){
// def map = rows.get(key)
def data_id = map._id.toString()
map.remove("_id")
request.add(new IndexRequest("trades", "_doc", data_id).source(map, XContentType.JSON));
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
}
@@ -1,6 +1,5 @@
package com.fangjia.script.district
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
@@ -9,11 +8,20 @@ import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* Created by dxy on 2016/12/15.
*/
@@ -26,6 +34,10 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
private static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
private static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
public void process(Object args) {
def cities = [
@@ -47,7 +59,7 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
"图木舒克","五家渠","济源","仙桃","潜江","天门","景德镇", "神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水",
"保亭","琼中","北屯","铁门关","双河","可克达拉"
]
cities = ["ÎÞÎý"]
cities = ["鹤岗"]
println cities
for(String city : cities) {
List<DBObject> districts = findDistrictByCity(city)
@@ -63,14 +75,14 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
for(DBObject district : districts) {
String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city")
long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0
if(district.get('alias')) {
for (String ali : district.get('alias')) {
if(ali) {
String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
}
}
}
// if(district.get('alias')) {
// for (String ali : district.get('alias')) {
// if(ali) {
// String skey = ali.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
// sellCount += (sellCountMap.get(skey) ? sellCountMap.get(skey) : 0)
// }
// }
// }
sellCountsList << (int)sellCount
}
sellCountsList = sellCountsList.unique()
@@ -219,67 +231,74 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
}
private def findDistrictHouseCount(String city, List<DBObject> districts, Date date) {
def regionDistrictMap = [:]
for(DBObject dbo : districts) {
def names = []
String n = DataUtil.escapeSolr(dbo.get('name'))
if(n) {
names << n
}
if(dbo.get("alias")) {
for(String ali : dbo.get("alias")) {
ali = DataUtil.escapeSolr(ali)
if(ali) {
names << ali
}
}
}
String region = dbo.get('region')
if(!regionDistrictMap.containsKey(region)) {
regionDistrictMap.put(region, names)
}else {
def list = regionDistrictMap.get(region)
list.addAll(names)
regionDistrictMap.put(region, list)
}
}
def startDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -1)),"yyyyMMdd")
def endDate = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(date),"yyyyMMdd")
Map<String, Long> monthMap = findAllCommHouse(city, districts, startDate.toInteger(), endDate.toInteger())
Map<String, Long> halfMap = new HashMap<String, Long>();
Map<String, Long> monthMap = new HashMap<String, Long>();
for(String region : regionDistrictMap.keySet()) {
def nameGroups = []
int len = 0
def nameList = []
for(String name : regionDistrictMap.get(region)) {
if(len >= 600 || nameList.size() >= 100) {
nameGroups << new ArrayList(nameList);
nameList.clear()
len = 0
}
len += name.length()
nameList << name
}
if(nameList.size() > 0) {
nameGroups << new ArrayList(nameList);
}
def startDate2 = DateUtil.formatByPattern(DateUtil.getFirstDayOfMonth(DateUtil.addMonthToDate(date, -6)),"yyyyMMdd")
Map<String, Long> halfMap = findAllCommHouse(city, districts, startDate2.toInteger(), endDate.toInteger())
println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
for(List<String> names : nameGroups) {
def halfRegionResult = [:]
def monthRegionResult = [:]
try {
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
}catch (e) {
sleep(10 * 1000)
halfRegionResult = facetDistrictCount(city, region, names, date, 7)
monthRegionResult = facetDistrictCount(city, region, names, date, 2)
}
halfMap.putAll(halfRegionResult)
monthMap.putAll(monthRegionResult)
}
}
// def regionDistrictMap = [:]
// for(DBObject dbo : districts) {
// def names = []
// String n = DataUtil.escapeSolr(dbo.get('name'))
// if(n) {
// names << n
// }
// if(dbo.get("alias")) {
// for(String ali : dbo.get("alias")) {
// ali = DataUtil.escapeSolr(ali)
// if(ali) {
// names << ali
// }
// }
// }
//
// String region = dbo.get('region')
// if(!regionDistrictMap.containsKey(region)) {
// regionDistrictMap.put(region, names)
// }else {
// def list = regionDistrictMap.get(region)
// list.addAll(names)
// regionDistrictMap.put(region, list)
// }
// }
//
//
// for(String region : regionDistrictMap.keySet()) {
// def nameGroups = []
// int len = 0
// def nameList = []
// for(String name : regionDistrictMap.get(region)) {
// if(len >= 600 || nameList.size() >= 100) {
// nameGroups << new ArrayList(nameList);
// nameList.clear()
// len = 0
// }
// len += name.length()
// nameList << name
// }
// if(nameList.size() > 0) {
// nameGroups << new ArrayList(nameList);
// }
//
// println city + "\t" + region + "\t" + regionDistrictMap.get(region).size() + "/" + nameGroups.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
// for(List<String> names : nameGroups) {
// def halfRegionResult = [:]
// def monthRegionResult = [:]
// try {
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
// }catch (e) {
// sleep(10 * 1000)
// halfRegionResult = facetDistrictCount(city, region, names, date, 7)
// monthRegionResult = facetDistrictCount(city, region, names, date, 2)
// }
// halfMap.putAll(halfRegionResult)
// monthMap.putAll(monthRegionResult)
// }
// }
return [halfMap, monthMap];
}
@@ -352,6 +371,106 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
return ret
}
def findAllCommHouse(city, districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = findHousesByDistrict(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(count){
String key = rMap.get("name") + "_" + rMap.get("region") + "_" + rMap.get("city")
retMap.put(key, count)
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+count
}
} catch (Exception e) {
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return retMap
}
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias) {
int retryCount = 0;
boolean success = false;
def count = 0
while (retryCount < MAX_TRY_COUNT && !success) {
try {
count = findHousesByMdate(city, region, cat, startDate, endDate, name, alias)
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
// 这里可以选择在重试之前等待一段时间
Thread.sleep(3000); // 例如等待1秒
}
}
if (!success) {
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
}
return count;
}
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
static main(args) {
runBatch(new DistrictDealSellRentRankBatch(), args)
}
@@ -20,7 +20,7 @@ class AbcEvaluationRowsBatch extends BaseBatch{
static final def fjtSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
static main(args) {
@@ -29,7 +29,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def charset = "GBK"
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
@@ -23,7 +23,7 @@ class FzDataResultStat {
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
@@ -16,7 +16,7 @@ import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
import com.fangjia.core.util.HttpRequest
@@ -169,7 +169,7 @@ DingDingMessageUtil.sendTextMessage(access_token, s)
//
//
//def count(name, sDate){
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
// def q = new BasicDBObject()
// q.put("city","¸£ÖÝ")
// q.put("s_date", sDate)
@@ -21,7 +21,7 @@ class ThirdPriceExportBatch extends BaseBatch{
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
@@ -16,7 +16,7 @@ import java.util.concurrent.CountDownLatch
class ThirdTest {
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
@@ -25,7 +25,7 @@ class TradeExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static def districtMap = [:]
@@ -30,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger
class CzBankStat {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "mongo.db.fangjia.com")
def static final DBCollection CZLOG = MongoUtils.getCollectionByName("fangjia_log", "cz_log2", "192.168.0.136")
static main(args) {
def task = new CzBankStat();
@@ -16,7 +16,7 @@ import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
class DealPriceProcess {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def total = 0
static main(args) {
@@ -17,7 +17,7 @@ import java.util.concurrent.Executors
class DistrictDetailBatch {
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "192.168.0.43")
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -40,7 +40,7 @@ class EsHouseStat {
dataSource.setProperties(connectionProperties)
}
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
public static void main(String[] args){
@@ -16,8 +16,8 @@ import java.sql.ResultSet
class FZDataExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
class FZHouseEvaluation {
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
def regionMap = [:]
@@ -1,5 +1,6 @@
package com.fangjia.script.gj
import com.fangjia.script.common.DataHandleUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
@@ -12,12 +13,13 @@ import com.fangjia.script.util.ExcelUtils;
class FangjiatongUserStat {
public static final DBCollection FJT_USER_COLL = MongoUtils.getCollectionByName("fangjiatong_log", "utm_gif_log", "222.73.242.42", 52002)
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
public static void main(String[] args) {
def task = new FangjiatongUserStat();
task.run();
// task.run();
task.stat()
}
def run(){
@@ -60,4 +62,59 @@ class FangjiatongUserStat {
return ret;
}
def stat(){
def users = findUses(12)
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
def rets = []
def header = ["用户id","支行名称", "备注名称"]
months.each{m->
header << m.substring(0, 7)
}
rets << header
for(def m : users){
def ret = []
ret << m.user_id
ret << m.name
ret << m.org_name
for(def i=1;i<months.size();i++){
def evalCount = findEvaluationCount(m.user_id, months[i-1], months[i])
ret << evalCount
}
for(def i=1;i<months.size();i++){
def downCount = findTradesCount(m.user_id, months[i-1], months[i])
ret << downCount
}
rets << ret
}
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019")
}
def findUses(role){
def sql = "select u.id,ud.name,ud.org_name from (SELECT * from users WHERE role=12) u left join users_detail ud on u.id=ud.id ";
def rets = [];
onLineSqlInstance.eachRow(sql, {
rets << ["user_id":it.id, "name":it.name, "org_name":it.org_name]
})
return rets
}
def findEvaluationCount(user_id, startDate, endDate){
def sql = "select count(*) as count from evaluation where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
def count = 0;
onLineSqlInstance.eachRow(sql, {
count = it.count
})
return count
}
def findTradesCount(user_id, startDate, endDate){
def sql = "select count(*) as count from user_trades where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
def count = 0;
onLineSqlInstance.eachRow(sql, {
count = it.count
})
return count
}
}
@@ -22,9 +22,6 @@ class FiledStrand {
def updateIndex = 0;
static main(args) {
// def task = new FiledStrand();
// task.run();
// task.updateOpenDate();
@@ -271,7 +268,7 @@ class FiledStrand {
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=5000 && v<=10000000){
if(v>=1000 && v<=10000000){
return v.toString().toDouble().toInteger()
}
}
@@ -288,11 +285,25 @@ class FiledStrand {
return null
}
static def processLobbyHeight(height){
if(height){
def v = getValue(height)
if(v>=2.4 && v<=30){
return v.toString()
}
}
return null
}
static def processElevators(elevators){
if(elevators){
def v = getValue(elevators)
if(v>0 && v<=100){
return v.toString()
if(v instanceof Integer){
if(v>0 && v<=100){
return v.toString()
}
}
}
return null
@@ -317,24 +328,34 @@ class FiledStrand {
static def processStandardArea(area){
if(area){
area = area.toString().replace("£¬",",").replace("©O", "").replace("m", "").replace("/","")
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
return area
def v = getValue(area)
if(v>=500 && v<=20000){
return v.toString().toDouble().toInteger()
}
}
return null
}
static def processCarSiteRent(rent){
if(rent){
def v = getNumberRange(rent)
if(v){
if(v=="0"){
if(["-","0元/月","无","不详","未知","待查","免费元/月","/","0","*","待确认","undefined元/月","0元","0元每月","未知元/月","0.00元/月","未定","暂定","测算中","无元/月","未定元/月","另议元/月","面议元/月","面议元/月","有元/月","暂时未确定","0元(无停车位)","&nbsp;元/月"].contains(rent)){
return null
}else if(rent.contains("暂无") || rent.contains("车位不归") || rent.contains("待定") || rent.contains("暂定") || rent.contains("暂完善") || rent.contains("待审批") || rent.contains("不详") || rent.contains("未定价格") || rent.contains("中国电信") || rent.contains("面谈")){
return null
}else{
rent = rent.replace("元/月","元/月").replace(",","").replace("#","").replace(".00","")
if(rent.toString().length()>100){
return null
}else{
return v.toString()
}
if(rent instanceof Double && rent.toDouble()<100){
return null
}
if(rent instanceof Integer && rent.toInteger()<100){
return null
}
return rent
}
return null
}
return null
}
@@ -382,7 +403,7 @@ class FiledStrand {
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if(v>5 && v<=100000){
if(v>5 && v<=10000){
try {
return v.toString().toInteger();
}catch(Exception e){
@@ -533,12 +554,82 @@ class FiledStrand {
return null
}
static def processLevel(val){
def level = null
if (val) {
if(val.contains("超甲") || val.contains("5A") || val.contains("6A") || val.contains("顶级")){
level = "超甲级"
}else if(val.indexOf("甲")>-1 || val.contains("A")){
level = "甲级"
}else if(val.indexOf("乙")>-1 || val.contains("B")){
level = "乙级";
}else if(val.indexOf("丙")>-1 || val.contains("C")){
level = "丙级";
}
}
return level
}
static def processLoopLine(val){
def line = null
if (val) {
if(!val.contains("暂") && !val.contains("无")){
if(val.indexOf("环")>-1){
line = val
}
}
}
return line
}
static def processForeign(val){
def foreign = null
if (val) {
if(val.indexOf("是")>-1){
foreign = "是"
}else if(val.indexOf("否")>-1){
foreign = "否";
}
}
return foreign
}
static def processRegister(val){
def register = null
if (val) {
if(val.contains("不可以注册") || val.contains("不能注册")){
register = "否";
}else if(val.contains("可以注册")){
register = "是"
}else if(val.indexOf("是")>-1){
register = "是"
}else if(val.indexOf("否")>-1){
register = "否";
}
}
return register
}
static def processFitment(val){
def fitment = null
if(val){
if(val.indexOf("精装")>-1){
fitment = "精装"
}else if(val.indexOf("简单")>-1 || val.indexOf("简装")>-1 || val.indexOf("一般")>-1 || val.indexOf("普通")>-1){
fitment = "简装";
}else if(val.indexOf("毛坯")>-1){
fitment = "毛坯";
}
}
return fitment
}
static def processLngAndLat(val) {
if (val) {
if (val && val!="null") {
def map = [:]
def lng = DataHandleUtils.getDoubleRound(6, val.x)
def lat = DataHandleUtils.getDoubleRound(6, val.y)
def lng = DataHandleUtils.getDoubleRound(6, val.lon)
def lat = DataHandleUtils.getDoubleRound(6, val.lat)
Long gd_lat = lat * 10000000000L
Long gd_lng = lng * 10000000000L
map.lat = gd_lat
@@ -618,7 +709,8 @@ class FiledStrand {
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
if(v>=0.3 && v<=10){
v = DataHandleUtils.getDoubleRound(2, v.toString().toDouble())
if(v>=0.3 && v<=20){
return v.toString()
}
}
@@ -644,7 +736,7 @@ class FiledStrand {
static def processVirescencePercent(virescence_percent){
if(virescence_percent){
def v = getValue(virescence_percent)
if(v && v>15 && v<=70){
if(v && v>=10 && v<=70){
return v.toString()
}
}
@@ -788,6 +880,20 @@ class FiledStrand {
return null
}
static def processUndergroundFloor(floor){
def v = getValue(floor.toString());
if(v){
if(v>=1 && v<=5){
try {
return v.toString();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processRealUsePercent(percent){
def v = getValue(percent.toString());
if(v){
@@ -834,15 +940,40 @@ class FiledStrand {
static def processCompany(val){
if(val){
if(val.length()>20){
if(val.length()>20 || val.length()<4){
return null
}else{
if(val.contains("ÔÝÎÞ")){
if(["#NA","*"].contains(val)){
return null
}
if(val.contains("暂无") || val.contains("暂定")){
return null
}
if(val.contains("未知") || val.contains("不祥")){
return null
}
if(val.contains("待定") || val.contains("待详情")){
return null
}
if(val.contains("尚未公开")){
return null
}
if(val.contains("在线咨询")){
return null
}
if(val.contains("平米") || val.contains("元平月")){
return null
}
if(val == "ÎÞ"){
return null
}
if(val == "-"){
return null
}
if(val == "--"){
return null
}
if(val.isNumber()){
return null
}
@@ -863,7 +994,7 @@ class FiledStrand {
if (pattern.matcher(val).matches()) {
return null
}
return val
return val.replace("*","").replace("*","")
}
}
@@ -909,14 +1040,18 @@ class FiledStrand {
}
static def getNumber(def input) {
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
try{
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
}
}catch(Exception e){
println input+" error"
}
return null;
}
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
class FjwUserStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection USER_COLL = MongoUtils.getCollectionByName2("fangjia_coresite", "user", lineUrl)
@@ -18,7 +18,7 @@ import org.bson.Document;
import org.bson.conversions.Bson;
class HouseLouDongMatch {
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
@@ -22,7 +22,7 @@ class LandDetailCrawl {
DBCollection LAND_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "land_deal_price1", "192.168.0.235", 27018)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", lineUrl)
public static final DBCollection OFF_FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", offlineUrl)
@@ -14,10 +14,10 @@ import org.apache.lucene.queryparser.classic.QueryParser
class LogStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new LogStat()
@@ -21,8 +21,8 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.GaoJuan)
class NoHouseDistrictStatBatch extends BaseBatch{
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
@@ -13,7 +13,7 @@ class PingAnAddressLevalTest {
String token = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB";
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "mongo.db.fangjia.com")
static main(args) {
def task = new PingAnAddressLevalTest()
task.run();
@@ -0,0 +1,62 @@
package com.fangjia.script.gj
import groovy.json.JsonOutput
class ProxyHttpUtils {
static String proxyHost = 'tunnel2.qg.net';
static Integer proxyPort = 14186;
static String proxyUsername = '113BBC06';
static String proxyPassword = '3B2FBD05B0AF';
static String getProxyResult(targetUrl, data){
// 将 JSON 对象转换为字符串
String jsonData = new JsonOutput().toJson(data);
// 创建代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
// 创建 URL 对象
URL url = new URL(targetUrl);
// 创建连接对象
URLConnection connection = url.openConnection(proxy);
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST");
// 允许输出流
connection.setDoOutput(true);
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json");
// 发送请求参数
OutputStream outputStream = connection.getOutputStream();
outputStream.write(jsonData.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
// 读取响应
int responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
response.append(line);
}
reader.close();
return response.toString();
} else {
println "HTTP request failed with response code ${responseCode}";
}
return null;
}
}
@@ -329,7 +329,7 @@ def findDistrictPrice(def city, def region){
q.put("region", region);
}
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
def count = districtStatConn.count(q)
return count;
}
@@ -343,7 +343,7 @@ def findDistrictPrice(def city, def region){
q.put("region", region);
}
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
def count = districtStatConn.count(q)
return count;
}
@@ -6,7 +6,7 @@ import com.mongodb.DBCollection
class RegionBlockSetting {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName2("fangjia", "region_block", lineUrl)
static main(args) {
@@ -17,8 +17,8 @@ import com.mongodb.DBObject
class SynchDistrictStatBatch extends BaseBatch{
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
class UpdateThirdPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col();
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -14,7 +14,7 @@ import groovy.sql.Sql
class ApiUserCountAnalyBatch extends BaseBatch{
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
@@ -10,7 +10,7 @@ import com.mongodb.DBCollection
class CcbCityPriceStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -104,6 +104,43 @@ class FileUtils {
sb.setLength(0)
}
out.close();
return sb.toString().getBytes(charset)
}
static def createEncryptFile(fileName, outputStr, isAppend){
OutputStream out = new FileOutputStream(fileName, isAppend);
out.write(outputStr.getBytes("utf-8"));
out.close();
}
static def createFileStringByCustomSuffix(ret, fileName, charset, suffix, lineSuffix){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
def sb = new StringBuffer();
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), suffix))
if(index<ret.size()){
sb.append(lineSuffix+"\n");
}else{
sb.append(lineSuffix);
}
}
}
return sb.toString()
}
@@ -20,7 +20,7 @@ import com.mongodb.DBCollection
*/
class PocTest2023 {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
@@ -15,7 +15,7 @@ import org.apache.commons.lang.math.RandomUtils
class DistrictSelectHouse {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictSelectHouse();
@@ -9,8 +9,8 @@ import com.mongodb.QueryOperators
class TestPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.QueryBuilders
class CityStatExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
@@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong
class CityStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
@@ -21,13 +21,13 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
//def citys = ["ÎÚ³ľÆë","²ý¼ª","°¢¿ËËÕ","ÒÁÀç","¿¦Ê²"]
@@ -0,0 +1,48 @@
package com.fangjia.script.gj.coresite
import java.net.HttpURLConnection
import java.net.URL
import java.io.OutputStream
// 目标服务器的URL
def targetUrlString = "http://data.zz.baidu.com/urls?site=sh.fangjia.com&token=UaKTemkcQ6I4YvSA"
// 需要POST的URL数据
def postData = "https://sh.fangjia.com/zoushi?cityName=%E4%B8%8A%E6%B5%B7&keyword=%E5%A4%A9%E5%B1%B1%E6%96%B0%E8%8B%91"
// 创建URL对象
URL url = new URL(targetUrlString)
// 打开连接
HttpURLConnection connection = (HttpURLConnection) url.openConnection()
connection.setDoOutput(true) // 表明我们将发送POST请求的输出数据
connection.setRequestMethod("POST") // 设置请求方法为POST
connection.setRequestProperty("User-Agent", "curl/7.12.1") // 设置User-Agent,尽管这在实际应用中可能并非必需
connection.setRequestProperty("Content-Type", "text/plain") // 设置内容类型,因为我们将发送纯文本数据
connection.setRequestProperty("Content-Length", Integer.toString(postData.getBytes().length)) // 设置内容长度
// 发送POST数据
//try (OutputStream os = connection.getOutputStream()) {
// byte[] input = postData.getBytes("utf-8")
// os.write(input, 0, input.length)
//}
outputStream = connection.getOutputStream();
outputStream.write(postData.getBytes());
outputStream.flush();
outputStream.close();
// 获取响应码
int responseCode = connection.getResponseCode()
println "POST Response Code :: $responseCode"
// 可选:读取响应内容
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"))
StringBuilder response = new StringBuilder()
String responseLine
while ((responseLine = reader.readLine()) != null) {
response.append(responseLine.trim())
}
println "Response Body :: ${response.toString()}"
// 关闭连接
connection.disconnect()
@@ -11,7 +11,7 @@ import com.mongodb.QueryOperators
class BestDistrictPriceExport {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new BestDistrictPriceExport();
@@ -0,0 +1,370 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.InPolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.json.JsonSlurper
import groovy.io.FileType
import org.bson.types.ObjectId
import java.awt.geom.Point2D
import java.util.regex.Pattern
class AddDevShapeData {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def devAreaMap = [:]
def fullCityMap = CommonDatautils.queryFullNameCity()
def results = new HashSet()
def existList = new HashSet()
static main(args) {
def task = new AddDevShapeData();
task.run()
// task.streetInfo()
// task.addStreetToMapping()//通过乡镇街道的点是否都在开发区围栏中,判断是否将此乡镇加入映射表中
}
def addStreetToMapping(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def devShapes = findDevShapeByCity(fullCityName)
if(devShapes){
def streetShapes = findStreetShapeByCity(fullCityName)
addDevStreetToMapping(fullCityName,streetShapes, devShapes)
}
}
results.each {s->
println s
}
existList.each{s->
println s
}
println " 新增映射乡镇:"+results.size()+" 已存在乡镇:"+existList.size()
}
def addDevStreetToMapping(city,streetShapes, devShapes){
def addCount = 0
def existsCount = 0
for(def streetObj : streetShapes){
def coordinates = streetObj.shape.coordinates
for(def pointList : coordinates){
def objList = findDevInStreetRange(streetObj, pointList, devShapes)
if(objList){
for(def obj : objList){
def existsFlag = isExistStreetByDevName(obj.name, streetObj.street)
if(existsFlag){
existsCount++
existList << "[存在] "+obj.name+" "+ streetObj.street
}else{
addCount++
results << "[新增] "+obj.name+"#"+ streetObj.street
if(!streetObj.street){
println streetObj
}
}
}
}
}
}
}
def findDevInStreetRange(streetObj,pointList, devShapes){
def devList = []
def minDis =200
for(def devObj : devShapes){
def flagList = []
def distances = []
for(def pointArr : pointList){
// def flag = InsidePolygon.isInArea(pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble(), InsidePolygon.createPoint4(devObj.shape.coordinates))
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6,pointArr[0].toString().toDouble()), DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()));
def flag = InPolygon.isInArea(point, InPolygon.createPoint(devObj.shape.coordinates))
flagList << flag
if(!flag){
distances << findNearestDistances(pointArr, devObj.shape.coordinates)
}
}
def trueCount = statResult(flagList)
def pre = trueCount*100/flagList.size()
println streetObj.district+" "+streetObj.street +" "+devObj.name +" "+flagList.size()+" "+pre+"%"
if(pre>=100){
devList << devObj
}
// else{
// if(distances && pre>=50){
// def minCount = 0
// for(def d : distances){
// if(d <= minDis){
// minCount++
// }
// }
// def simlarPointPre = minCount*100/distances.size()
// if(simlarPointPre>=90){
// devList << devObj
// }
// println "不在多边型范围内的点,距离小于${minDis}米有:"+minCount+" 占比:"+simlarPointPre
// }
// }
}
return devList
}
def findNearestDistances(pointArr, coordinates){
def minDis = Integer.MAX_VALUE
for(def pointlist : coordinates){
for(def devPoint : pointlist){
def distance = GeoUtils.calculateDistance(pointArr[0], pointArr[1], devPoint[0], devPoint[1])
if(distance < minDis){
minDis = distance
}
}
}
return minDis
}
def statResult(rows){
def count = 0
for(def s : rows){
if(s){
count++
}
}
return count
}
def findStreetShapeByCity(city){
def list = []
def dbcurosr = administrative_mapping_coll.find(new BasicDBObject("city", city).append("type", "乡镇街道").append("shape", new BasicDBObject('$exists', true)))
for(def d : dbcurosr){
list << d
}
return list
}
def findDevShapeByCity(city){
def list = []
def dbcurosr = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("city", city).append('shape', new BasicDBObject('$exists',true)))
for(def d : dbcurosr){
list << d
}
return list
}
def isExistStreetByDevName(devName, street){
def list = dev_mapping_CONN.find(new BasicDBObject("name", devName).append("type","乡镇街道").append("township",street)).toList()
if(list){
return true
}
}
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
dir.eachFileRecurse(FileType.FILES) { file ->
process(file.absolutePath)
}
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
} else {
println "The specified path is not a directory."
}
}
def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d->
// if(!d['I']){
def name = d['E']
def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A']))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
if(obj){
if(obj.operator){
d['H'] = obj.operator
}
if(obj.operate_result){
d['I'] = obj.operate_result
}
if(obj.shape){
d['T'] = "有"
}
// if(obj.m_date){
// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
// }
// if(devAreaMap){
// devAreaMap.each {k,v->
// if(obj.name && k && obj.name.contains(k)){
// d['M'] = v.area_info
// d['N'] = v.count
// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
// }
// }
//
// }else{
// println name+" 空值"
// }
}
// }
})
}
def process(file){
def retMap = getNewLocations(file)
if(retMap && retMap.gaode.size()>0){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+retMap.name+'.*$', Pattern.CASE_INSENSITIVE)))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
if(obj){
for(def i=0;i<retMap.gaode.size();i++){
def udata = new BasicDBObject()
def resultMap = [:]
def bd_resultMap = [:]
resultMap.type = "polygon"
resultMap.coordinates = [retMap.gaode[i]]
bd_resultMap.type = "polygon"
bd_resultMap.coordinates = [retMap.baidu[i]]
if(i>0){
obj.removeField("_id")
obj.put("create_time", new Date())
obj.put("block", retMap.name+"-地块"+i)
obj.put("shape", resultMap)
obj.put("m_date", new Date())
obj.put("bd_shape",bd_resultMap)
obj.put("operator", "外包")
obj.put( "operate_result" , "已处理")
DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{
udata.put("shape", resultMap)
// udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "已处理")
udata.put("operator", "外包")
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
}
}
}
}
def getNewLocations(file){
def jsonObj = readJson(file)
def features = jsonObj.features
def gaode_locations = []
def baidu_locations = []
def name = null
def areaMap = [:]
if(features && features.size()==1){
name = features[0].properties.Name
def coordinates = features[0].geometry.coordinates
def index = 0
println file.toString()+" "+name +" 有 "+coordinates.size()+" 块"
areaMap.count = coordinates.size()
def areas = []
// if(name.contains("江苏常熟综合保税区")){
for(def pointList : coordinates){
index ++
def gaode_coordinates = []
def bd_coordinates = []
def points = null
if(pointList.size()>1){
points = pointList
}else{
points = pointList[0]
}
double area = com.fangjia.script.util.GeoHelper.calculateArea(points)
println name +" 地块"+index + "围栏面积是: ${area} 平方公里"
areas << area
def locationstr = ""
for(def arrs : points){
def gaode_map = GeoUtils.gps84ToGcj02(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
gaode_coordinates << [gaode_map.get("lng"), gaode_map.get("lat")]
//此高德转百度 有偏差
def bd_map = GeoUtils.gcj02ToBd09(gaode_map.get("lat"), gaode_map.get("lng"))
// bd_coordinates << [bd_map.get("lng"), bd_map.get("lat")]
// 高德转百度 通过接口有次数限制
// def baidu = BaiDuApiUtils.geoconv(gaode_map.get("lng"), gaode_map.get("lat"), 3)
// bd_coordinates << [baidu.x, baidu.y]
locationstr = arrs[0].toString()+","+arrs[1].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()), 6)
if(bdpoint){
bd_coordinates << bdpoint
}
}
//整个一串一起传超过4029会报错
// bd_coordinates = FangjiaClientUtils.convert(locationstr.substring(0, locationstr.length()-1), 6)
gaode_locations << gaode_coordinates
if(bd_coordinates){
baidu_locations << bd_coordinates
}else{
println name +" 转换有问题"
}
}
areaMap.area_info = areas.join("\n")
// }
}else{
println file +" 有多个"
}
devAreaMap.put(name, areaMap)
def retMap = [:]
retMap.name = name
retMap.gaode = gaode_locations
retMap.baidu = baidu_locations
return retMap
}
def readJson(def file){
def jsonSlurper = new JsonSlurper()
def jsonFile = new File(file)
def jsonText = jsonFile.text
def jsonObj = jsonSlurper.parseText(jsonText)
return jsonObj
}
def streetInfo(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx", {d->
def count = dev_mapping_CONN.count(new BasicDBObject("name", d['E']).append("type", "乡镇街道"))
d['L'] = count
})
}
}
@@ -0,0 +1,448 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.json.JsonSlurper
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//补充库中没有详细地址的小区
class AddDistrictAddress extends BaseBatch{
// def fullCityMap = CommonDatautils.queryFullNameCity()
// def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
//
// DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
// public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
//
//
// public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
//
// def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
// DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
//
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new AddDistrictAddress(), args)
}
@Override
public void process(Object args) {
// addNoAddressDist() //为不精确的地址通过地图补充地址
// mapOtherAddress() //有精确地址的寻找其它地址
// addressSelect()
addrUpdateToDB()
}
def addNoAddressDist(){
// def citys = CommonDatautils.findAllCity()
// def index = 0
// def allrets = []
// for(def c : citys){
// index ++
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
//
// def rets = []
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
// for(def d : noRoadInfoDists){
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
// }
// allrets.addAll(rets)
// if(allrets.size()>50000){
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
// allrets.clear()
// }
// }
updateDistAddr()
}
def mapOtherAddress(){
def citys = CommonDatautils.findAllCity()
def index = 0
def allrets = []
for(def c : citys){
index ++
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address", "addr_alias"])
def radInfoDists = haveRoadInfoDistricts(districts)
println c +" "+index +" 原始小区量:"+districts.size() +" 有详细地址量:"+radInfoDists.size()
radInfoDists = baiduMatch(c, radInfoDists)
radInfoDists = gaodeMatch(c, radInfoDists)
def rets = []
rets << ["id", "城市", "区域", "名称", "地址","地址别名", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
for(def d : radInfoDists){
def addr_alias_str = '';
if(d.addr_alias){
addr_alias_str = d.addr_alias.join(" ")
}
rets << [d._id.toString(), d.city, d.region, d.name, d.address, addr_alias_str,d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
}
allrets.addAll(rets)
if(allrets.size()>50000){
ExcelUtils.write("D:\\数据\\小区\\小区精确地址补充"+index+".xlsx", allrets, "补充地址")
allrets.clear()
}
}
}
def addressSelect(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-all-2.xlsx",{d->
if(d['G']){
d['O'] = AddressUtil.formatAddress(d['B'], d['G'])
}
if(d['I']){
d['P'] = AddressUtil.formatAddress(d['B'], d['I'])
}
})
}
def addrUpdateToDB(){
def updateDist = 0
def addAddrCount = 0
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\精确地址补充-入库-2.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def oldAddrAliasCount = 0
if(dist.addr_alias){
oldAddrAliasCount = dist.addr_alias.size()
}
def new_addr_alias = null
if(d['K'].contains("新增百度地址") || d['K'].contains("双地图新增")){
new_addr_alias = getNewAddressAlias(d['L'], dist.address, dist.addr_alias)
}
if(d['K'].contains("新增高德地址") || d['K'].contains("双地图新增")){
new_addr_alias = getNewAddressAlias(d['M'], dist.address, dist.addr_alias)
}
if(new_addr_alias){
updateDist++
println updateDist
addAddrCount = addAddrCount + new_addr_alias.size()-oldAddrAliasCount
def udata = new BasicDBObject()
udata.put("addr_alias", new_addr_alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
}
}
})
println "修改小区数:"+updateDist
println "新增地址数:"+addAddrCount
}
def getNewAddressAlias(new_addr, old_addr, dist_addr_alias){
def arrs = new_addr.split(";")
def addr_alias = []
if(dist_addr_alias){
addr_alias = dist_addr_alias
}
for(def address : arrs){
if(old_addr!=address && !addr_alias.contains(address)){
addr_alias << address
}
}
return addr_alias
}
def updateDistAddr(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("address", d['J'])
def addr_alias = dist.addr_alias
if(!dist.addr_alias){
addr_alias = []
}
if(d['K'] && !addr_alias.contains(d['K'])){
addr_alias << d['K']
}
if(addr_alias){
udata.put("addr_alias", addr_alias)
}
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
d['L'] = "已更新"
}
})
}
def noDetailRoadInfoDistricts(districts){
def rets = []
for(def d : districts){
if(d.address){
def flag = DataHandleUtils.isStandardRoad(d.address)
if(!flag){
rets << d
}
}else{
rets << d
}
}
return rets;
}
def haveRoadInfoDistricts(districts){
def rets = []
for(def d : districts){
if(d.address){
def flag = DataHandleUtils.isStandardRoad(d.address)
if(flag){
rets << d
}
}
}
return rets;
}
def baiduMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = baiduProcess(d, fullCityName)
if(retMap){
d.baidu_address = retMap.new_addr
d.baidu_old_addr = retMap.old_addr
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def baiduProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
for(def name : alias){
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
if(temp_address){
retMap.old_addr = temp_address
}
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
retMap.new_addr = nineAddrSplit(fullCityName, temp_address)
continue
}
}
return retMap
}
def nineAddrSplit(city, addr){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
def address = ""
if (jsonObject) {
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
address = jsonObject.get("street")
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
address = address + jsonObject.get("streetNumber")
}
}
return address
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def dbcursor = BAIDU_POI_COLL.find(q)
def address = null
for(def d : dbcursor){
address = d.addr
}
return address;
}
def gaodeMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = gaodeProcess(d, fullCityName)
if(retMap){
d.gaode_addr = retMap.new_addr
d.gaode_old_addr = retMap.old_addr
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def gaodeProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
def address = null
for(def name : alias){
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
if(temp_addr){
retMap.old_addr = temp_addr
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
// retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
retMap.new_addr = nineAddrSplit(fullCityName, temp_addr)
continue
}
}
}
return retMap
}
def findGaoDeDatas2(city, region, name){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
def body = """
{
"_source": {
"includes": ["cat", "address", "id","name"]
},
"query": {
"bool": {
"must": [
{
"term": {
"city" : "${city}"
}
},
{
"term": {
"district" : "${region}"
}
},
{
"match": {
"name": {"query": "${name}","operator": "and"}
}
}
// ,{
// "match": {
// "small_type": "住宅"
// }
// }
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def address = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
def d = hits[0]
if(d._source.cat.contains("住宅")){
def source = d._source
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
if(newName == name){
address = source.address
}
}
}
}
return address
}
}
@@ -30,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
@@ -55,7 +58,7 @@ class AddDistrictShapeBatch extends BaseBatch{
// }
// checkRangeCount()
synch100()
// synch100()
}
def baiduMatch(city,districts){
@@ -378,20 +381,21 @@ class AddDistrictShapeBatch extends BaseBatch{
def checkRangeCount(){
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
// def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
def dbcursor = administrative_mapping_coll.find(new BasicDBObject("type","ÏçÕò½ÖµÀ")).batchSize(100)
def count = 0
def index =0
for(def d : dbcursor){
index ++
if(d.location){
def coordinates = d.location.coordinates
if(d.shape){
def coordinates = d.shape.coordinates
if(coordinates.size()>1){
count++
println count +" "+ d.city +" "+d.name
println count +" "+ d.city +" "+d.street
}
}else{
println d.city + " "+d.name
println d.city + " "+d.street
}
if(index%1000==0){
@@ -0,0 +1,211 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.regex.Pattern
class AddrAlaisProcess {
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new AddrAlaisProcess()
// task.run()
// task.splitDist()
// task.addAddr()
task.nineApi()
// task.createResult()
// task.mergeResult()
}
def splitDist(){
def rows = ExcelUtils.read("D:\\数据\\数据建设\\小区\\全量小区地址及地址别名.xlsx",0,1)
println "原始总计:"+rows.size()
def rets = []
for(def d : rows){
def addr = d['F']
if(d['G']){
addr = addr +","+d['G']
}
def addrs = addr.split(",")
for(def s : addrs){
rets << [d['A'], d['B'], d['C'], d['D'], d['E'], s]
}
}
println "总计:"+rets.size()
FileUtils.batchCreateFileByCustomSuffix(rets, "D:\\数据\\数据建设\\小区\\有地址小区地址标准化(一行一个地址).bat", null, "utf-8", false, " ","");
}
def addAddr(){
def fileName = "D:\\数据\\数据建设\\小区\\无地址小区.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
// d['F'] = dist.address
if(dist.addr_alias){
d['G'] = dist.addr_alias.join(",")
}
}
})
}
def mergeResult(){
def rets = ExcelUtils.read("D:\\数据\\数据建设\\小区\\ai-结果-0221.xlsx",0,1)
def retMap = [:]
for(def r : rets){
def num = r['B'].toString().toDouble().toInteger()
def list = retMap.get(num)
if(!list){
list = new HashSet()
retMap.put(num, list)
}
if(r['C']){
list << r['C']
}
}
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
ExcelUtils.modify(fileName,{d->
def num = d['A'].toString().toDouble().toInteger()
def aiList = retMap.get(num)
if(aiList){
d['K'] = aiList.join(",")
}
})
}
def createResult(){
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
def results = []
// results << ["ID", "城市", "地址"]
ExcelUtils.modify(fileName,{d->
def str = d['C']
def addressList = str.split(",")
for(def addr : addressList){
results << [d['A'].toString().toDouble().toInteger(), d['B'], addr]
}
})
println results.size()
FileUtils.batchCreateFileByCustomSuffix(results, "D:\\数据\\数据建设\\小区\\ai-待支行.bat", null, "utf-8", false, " ","");
}
def nineApi(){
def i = 0;
while(i<20) {
def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName, {d->
def addrMap = nineAddrSplit(d['D'], d['G'])
def addrList = []
if(addrMap && addrMap.address){
d['H'] = addrMap.road
d['I'] = addrMap.road_num
}
if(d['H']){
def data = fullCityMap.get(d['D'])
if(data){
def roadObj = findBaiduDatas( data.city, d['H'])
if(roadObj){
d['J'] = "百度存在"
}
}
}
// if(d['F']){
// def addrs = d['F'].split(",")
// def addr_alias = []
// for(def addr : addrs){
// def map = nineAddrSplit(d['B'], addr)
// if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
// if(!addrList.contains(map.address)){
// addrList << map.address
// addr_alias << map.address
// }
// }
// }
// d['I'] = addr_alias.join(",")
// }
})
println fileName +" 完成 "
}
i++
}
}
def nineAddrSplit(city, addr){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr)
def address = ""
def map = [:]
if (jsonObject) {
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
map.province = jsonObject.get("province")
}else{
map.province = ""
}
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
map.city = jsonObject.get("city")
}else{
map.city = ""
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
map.district = jsonObject.get("district")
}else{
map.district = ""
}
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
address = jsonObject.get("street")
map.road = address
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
address = address + jsonObject.get("streetNumber")
map.road_num = jsonObject.get("streetNumber")
}
map.address = address
}
return map
}
def run(){
def citys = CommonDatautils.findAllCity()
def rows = []
citys.each{c->
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","address", "addr_alias"])
println c +" "+districts.size()
def num = 0
for(def d : districts){
num++
def row = [num, d._id.toString(), d.city, d.region, d.name, d.address]
if(d.addr_alias){
row << d.addr_alias.join(",")
}
rows << row
}
}
// ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区")
FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-20250402.bat", null, "utf-8", false, "#","");
}
def findBaiduDatas(city, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", name)
q.put('std_tag', Pattern.compile('^道路'))
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
}
@@ -4,6 +4,7 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -30,7 +31,10 @@ class AutoAddDistrictBatch extends BaseBatch{
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
runBatch(new AutoAddDistrictBatch(), args)
// runBatch(new AutoAddDistrictBatch(), args)
def task = new AutoAddDistrictBatch();
task.checkMap()
}
@Override
@@ -38,7 +42,7 @@ class AutoAddDistrictBatch extends BaseBatch{
//查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
@@ -48,11 +52,10 @@ class AutoAddDistrictBatch extends BaseBatch{
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
fileName = "D:\\数据\\add_dist_info_1.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
}
@@ -61,6 +64,11 @@ class AutoAddDistrictBatch extends BaseBatch{
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中已存在"
}else{
dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中别名中存在"
}
}
})
}
@@ -154,10 +162,10 @@ class AutoAddDistrictBatch extends BaseBatch{
}
def findGaoDeDatas(city, region, name){
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
if(d.name == name){ //&& d.shape
obj = d
}
})
@@ -250,8 +258,37 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def checkMap(){
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\写字楼存在性判断-test.xlsx",{d->
def gaode_obj = findGaodeCount(d['B'], d['C'], d['D'])
if(gaode_obj){
d['H'] = "是"
d['I'] = gaode_obj.type
}else{
def baidu_obj = findBaiduCount(d['B'], d['C'], d['D'])
if(baidu_obj){
d['E'] = "是"
d['F'] = baidu_obj.std_tag
}
}
// def baidu_obj1 = findBaiduCount(d['B'], d['G'], d['H'])
// if(baidu_obj1){
// d['M'] = "是"
// }
// def gaode_obj1 = findGaodeCount(d['B'], d['G'], d['H'])
// if(gaode_obj1){
// d['N'] = "是"
// }
// def score = CosineSimilarAlgorithm.getSimilarityRatio(d['D'], d['H']);
// d['O'] = score
})
}
}
@@ -0,0 +1,563 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CopyOnWriteArraySet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AutoAddOfficeBatch extends BaseBatch {
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206", 27020)
//235为房价网内部计算均价的库
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
// runBatch(new AutoAddOfficeBatch(), args)
def task = new AutoAddOfficeBatch()
// task.repeatOffice()
task.repeatDist()
}
@Override
void process(Object args) {
// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
// def newName = "开放广场".replaceAll(pattern, "")
// println newName
// findAddOfficeDatas() //查找需要新增的写字楼数据
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
// addOffice()//新增写字楼
}
def findAddOfficeDatas() {
def citys = CommonDatautils.findAllCity()
def allAddOffices = []
allAddOffices << ["地图ID", "省", "市", "区", "写字楼名称", "地址", "经度", "纬度", "匹配城市", "匹配区域", "匹配名称", "匹配方式"]
for (def c : citys) {
def cityObj = fullCityMap.get(c)
if (cityObj) {
def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼")
officeMaps = filterOffice(officeMaps)
if (officeMaps) {
def offices = matchOffice(officeMaps)
allAddOffices.addAll(offices)
}
}
}
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
}
def checkInBaiduMapOffice() {
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx", { d ->
if (!d['L']) {
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
if (obj) {
d['L'] = "百度存在"
d['M'] = obj.std_tag.toString()
}
}
}, 0)
}
def filterOffice(officeMaps) {
def offices = []
def keys = []
for (def o : officeMaps) {
if (!o) {
continue
}
if (o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")) {
println "[ 过滤 ] " + o.city + " " + o.name
continue
}
if (o.name.contains("路写字楼")) {
println "[ 过滤 ] " + o.city + " " + o.name
continue
}
def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
def newName = o.name.replaceAll(pattern, "")
o.name = newName
def keyName = o.city + "#" + o.district + "#" + o.name
if (!keys.contains(keyName)) {
offices << o
keys << keyName
}
}
println "原:" + officeMaps.size() + " 新:" + offices.size()
return offices
}
def matchOffice(offices) {
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
for (def i = 0; i < offices.size(); i++) {
final def nmd = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
addCount.addAndGet(1)
def office = matchDB(nmd)
if (!office) {
office = matchApi(nmd)
}
if (office) {
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, office.city, office.region, office.name, office.match_way]
println addCount.get() + "/" + total + " [success] " + office.get("region") + " " + office.get("name") + " " + office.match_way
} else {
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, "", "", "", ""]
println addCount.get() + "/" + total + " [fail] " + nmd.city + " " + nmd.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def matchDB(nmd) {
def simpleCity = RegionData.formatCity(nmd.city)
def simpleDistrict = RegionData.formatRegion(simpleCity, nmd.district)
if (simpleCity && simpleDistrict) {
def office = null
def rets = CommonDatautils.findDistrictList(simpleCity, simpleDistrict, nmd.name, "office")
if (rets) {
if (rets.size() >= 1) {
office = rets[0]
}
} else {
rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office")
if (rets && rets.size() >= 1) {
office = rets[0]
} else {
rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district")
if (rets && rets.size() >= 1) {
office = rets[0]
}
}
}
if (office) {
office.match_way = "数据库"
}
return office
}
return null
}
def matchApi(nmd) {
def address = nmd.city
if (nmd.district) {
address = address + nmd.district
}
address = address + nmd.name
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
if (!jsonObject) {
if (DataUtil.isPreciseAddress(nmd.address)) {
address = nmd.city
if (nmd.district) {
address = address + nmd.district
}
address = address + nmd.address
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
}
}
if (jsonObject) {
def udata = [:]
udata.put("_id", jsonObject.get("id"))
udata.put("city", jsonObject.get("city"))
udata.put("region", jsonObject.get("district"))
udata.put("name", jsonObject.get("name"))
udata.put("match_type", jsonObject.get("propertyType"))
udata.put("match_credit", jsonObject.get("credit"))
udata.put("match_way", "接口")
return udata
}
return null
}
def repeatOffice() {
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << ["ID", "城市", "区域", "名称", "地址", "可疑ID", "可疑城市", "可疑区域", "可疑名称", "可疑地址", "相似度", "距离"]
for (def c : citys) {
def offices = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "address", "lat", "lng"])
def rets = checkRepeatOfficeBatch(offices)
println c + " " + offices.size()
allrets.addAll(rets)
}
ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据")
}
def repeatDist(){
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
def index = 0
for (def c : citys) {
index++
if(["宁波","无锡","青岛"].contains(c)){
continue
}
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
def rets = checkRepeatDistrictBatch(c, districts)
println c + " " + districts.size()
allrets.addAll(rets)
println "${index}"+ c +" 运A行完成 "+rets.size()
if (CommonDatautils.isWindow()) {
ExcelUtils.write("D://数据//小区/重复可疑小区-${c}.xlsx", allrets, "数据")
}else{
ExcelUtils.write("/home/goojia/gj/重复可疑小区-${c}.xlsx", allrets, "数据")
}
}
}
def checkRepeatDistrictBatch(city,districts) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityId = cityCodeDataMap.get(city)
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def total = districts.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for (def i = 0; i < total; i++) {
final def o1 = districts[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try {
addCount.addAndGet(1)
// if(!o1.name.contains("四公司")){
// continue
// }
if(!sets.contains(o1._id.toString())){
def similarRets = findListSimilar(o1, districts)
if (similarRets) {
def newPriceMap1 = fjwPriceMap.get(o1._id.toString())
def new_price1 = null
if(newPriceMap1){
new_price1 = newPriceMap1.calc_type
}
def ailas_str1 = ""
if(o1.alias){
ailas_str1 = o1.alias.join(",")
}
def rows = []
for(def similar : similarRets){
if (!sets.contains(similar._id.toString())) {
sets.add(similar._id.toString())
def ailas_str = ""
if(similar.alias){
ailas_str = similar.alias.join(",")
}
def newPriceMap = fjwPriceMap.get(o1._id.toString())
def new_price = null
if(newPriceMap){
new_price = newPriceMap.calc_type
}
rows << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
// rets << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
}
}
if(rows){
sets.add(o1._id.toString())
rets << [ o1.city, o1.region, o1.name, ailas_str1,new_price1, o1.address, "主", "主", "主"]
rets.addAll(rows)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
// latch.await()
return rets
}
def checkRepeatOfficeBatch(offices) {
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for (def i = 0; i < total; i++) {
final def o1 = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
addCount.addAndGet(1)
def maxObj = findOneSimilarOffice(o1, offices)
if (maxObj) {
if (!sets.contains(o1._id.toString())) {
sets.add(maxObj._id.toString())
rets << [o1._id.toString(), o1.city, o1.region, o1.name, o1.address, maxObj._id.toString(), maxObj.city, maxObj.region, maxObj.name, maxObj.address, maxObj.score, maxObj.distance]
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findListSimilar(o1, datas){
def rets = new CopyOnWriteArrayList()
def ids = []
ids << o1._id.toString()
for (def o2 : datas) {
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region) {
// if(!o2.name.contains("四公司")){
// continue
// }
def name1 = o1.name//.replace("小区","").replace("别墅","")
def name2 = o2.name//.replace("小区","").replace("别墅","")
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
def addrScore = null
if(o1.address && o2.address){
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
}
def str = null
if(o1.name.length() > o2.name.length()){
str = o1.name.replace(o2.name, "")
}else{
str = o2.name.replace(o1.name, "")
}
if (nameScore > 0.6 && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
o2.nameScore = nameScore
o2.addrScore = addrScore
o2.score = nameScore
rets << o2
ids << o2._id.toString()
def maxObj = findMaxSimilar(ids, o2, datas)
while(maxObj){
maxObj.nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(maxObj.name, o1.name);
if(o1.address && maxObj.address){
maxObj.addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, maxObj.address);
}
maxObj.score = maxObj.nameScore
rets << maxObj
ids << maxObj._id.toString()
maxObj = findMaxSimilar(ids, maxObj, datas)
}
}
}
}
rets.sort{a,b->
def a1 = a.score
def b1 = b.score
return b1.compareTo(a1)
}
if (rets.size()>2) {
return rets[0..2]
}
return rets
}
def findMaxSimilar(ids, o1, datas){
def maxObj = null
def maxScore = -1000
for (def o2 : datas) {
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region && !ids.contains(o2._id.toString())) {
def name1 = o1.name//.replace("小区","").replace("别墅","")
def name2 = o2.name//.replace("小区","").replace("别墅","")
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
def addrScore = null
if(o1.address && o2.address){
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
}
def str = null
if(o1.name.length() > o2.name.length()){
str = o1.name.replace(o2.name, "")
}else{
str = o2.name.replace(o1.name, "")
}
if (nameScore > 0.6 && nameScore>maxScore && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
maxObj = o2
o2.nameScore = nameScore
o2.addrScore = addrScore
o2.score = nameScore
}
}
}
return maxObj
}
def findOneSimilarOffice(o1, offices) {
def maxScore = 0
def maxObj = null
for (def o2 : offices) {
if (o1._id.toString() != o2._id.toString()) {
double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name);
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address, o2.address);
if (nameScore + addrScore > maxScore) {
maxScore = nameScore + addrScore
maxObj = o2
}
}
}
def distance = -1000
if (o1.lat && maxObj && maxObj.lat) {
distance = GeoHelper.GetDistance(o1.lat / 10000000000.0, o1.lng / 10000000000.0, maxObj.lat / 10000000000.0, maxObj.lng / 10000000000.0)
}
if (maxObj) {
maxObj.distance = distance
maxObj.score = maxScore
return maxObj
}
return null
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def addOffice(){
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
if(d['L']!='人工删除'){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
if(city && region){
def locationstr = d['G'].toString()+","+d['H'].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
def udata = new BasicDBObject()
if(bdpoint){
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
}
udata.put("city", city)
udata.put("region", region)
udata.put("name", d['E'])
udata.put("address", d['I'])
udata.put("cat", "office")
udata.put("status", 0)
udata.put("c_date",new Date())
udata.put("m_date",new Date())
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
def estate_type = null
if(d['M'] && d['M'].contains("园区")){
estate_type = "创意园区"
}
if(!estate_type && d['E'].contains("创意")){
estate_type = "创意园区"
}
if(estate_type){
udata.put("estate_type2", estate_type)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map", markMap)
DbUtils.seaweed.save(udata)
println d['E'] +" "+udata._id.toString()
}
}
})
}
def findNewFjwAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance235.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def findNewFjwAvgPriceList(def cityid, date){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr,gujia_bds_id from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujiabds_clc_yrmo_val=${date}"
def retMap = [:]
pgsqlInstance235.eachRow(sql, { r ->
def map = [:]
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
retMap.put(r.gujia_bds_id, map)
})
return retMap
}
}
@@ -0,0 +1,268 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class BusinessQuarterProcess {
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def fullCityMap = CommonDatautils.queryFullNameCity()
DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020)
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new BusinessQuarterProcess()
task.run()
}
def run(){
// synchBusinessData()
// updateBusinessQuarter()
// addRegion()
synchBusinessToEs()
}
def addRegion(){
def list = CommonDatautils.queryRangeByCat("区域")
println list.size()
def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2")))
for(def d : dbcursor){
if(d.district){
continue
}
for(def regionShape : list){
if(d.short_city!=regionShape.city_short){
continue
}
def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates))
if(flag){
def udata = new BasicDBObject()
udata.put("district", regionShape.mzb_district_name)
udata.put("short_district", regionShape.district_short)
bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
break
}
}
}
}
def synchBusinessData(){
def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList()
synchMongoProcess(datas)
}
def synchMongoProcess(datas){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
createData(d)
println d.city +" "+totalCount.get() +"/"+datas.size()
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def createData(d){
def row = new BasicDBObject()
row.put("_id", d._id)
row.put("short_province", d.province)
def short_city = RegionData.formatCity(d.city)
row.put("short_city", short_city)
row.put("name", d.name)
row.put("bd_location", ["lon":DataHandleUtils.getDoubleRound(6,d.longitude), "lat":DataHandleUtils.getDoubleRound(6,d.latitude)])
def data = fullCityMap.get(short_city)
if(data){
row.put("province", data.province)
row.put("city", data.city)
}else{
println d.city + " 没找到简称"
}
def gdPoints = FangjiaClientUtils.convert(d.longitude+","+d.latitude, 3)
if(gdPoints){
row.put("location", ["lon":DataHandleUtils.getDoubleRound(6,gdPoints[0]), "lat":DataHandleUtils.getDoubleRound(6,gdPoints[1])])
}
def border = d.border
def bd_shapes = []
def gd_shapes = []
if(border){
def arrs = border.split(";")
if(border.contains("|")){
println d._id.toString()+" 可能含多个围栏"
}
for(def i=0;i<arrs.size();i++){
def str = arrs[i]
def arr = str.split(",")
def bd_lng = DataHandleUtils.getDoubleRound(6, arr[0].toString().toDouble())
def bd_lat = DataHandleUtils.getDoubleRound(6, arr[1].toString().toDouble())
if(!bd_shapes.contains([bd_lng, bd_lat]) ){
bd_shapes << [bd_lng, bd_lat]
}
if(bd_shapes.contains([bd_lng, bd_lat]) && i+1==arrs.size() && bd_shapes[0]==[bd_lng, bd_lat]){
bd_shapes << [bd_lng, bd_lat]
}
def tempGdPoints = FangjiaClientUtils.convert(bd_lng+","+bd_lat, 3)
if(tempGdPoints){
def gdlist = [DataHandleUtils.getDoubleRound(6,tempGdPoints[0]), DataHandleUtils.getDoubleRound(6,tempGdPoints[1])]
if(!gd_shapes.contains(gdlist)) {
gd_shapes << gdlist
}
if(gd_shapes.contains(gdlist) && i+1==arrs.size() && gd_shapes[0]==gdlist){
gd_shapes << gdlist
}
}
}
if(bd_shapes[0] != bd_shapes[bd_shapes.size()-1]){
bd_shapes << bd_shapes[0]
gd_shapes << gd_shapes[0]
}
}
def gdMap = [:]
gdMap.put("type" , "polygon")
gdMap.put("coordinates", [gd_shapes])
row.put("shape", gdMap)
def bdMap = [:]
bdMap.put("type" , "polygon")
bdMap.put("coordinates", [bd_shapes])
row.put("bd_shape", bdMap)
row.put("c_date", new Date())
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
}
def updateBusinessQuarter(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
if(["上海","北京"].contains(c)){
continue
}
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
businessProcess(c, districts, cityBusinessList)
}
}
def businessProcess(c,districts, cityBusinessList){
def updateIndex = 0
for(def d : districts){
if(!d.business_quarter){
def businessQuarter = findBusinessByRange(d, cityBusinessList)
if(businessQuarter){
updateIndex ++
def udata = new BasicDBObject()
udata.put("business_quarter", businessQuarter)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
println c +" "+ districts.size() +" "+updateIndex
}
def findBusinessByRange(d, cityBusinessList){
def name= null
for(def shapeObj : cityBusinessList){
if(d.lng){
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(shapeObj.shape.coordinates))
if(flag){
name = shapeObj.name
break
}
}
}
return name
}
def synchBusinessToEs(){
def dbcursor = bk_business_coll.find(new BasicDBObject())
def rows = []
def index = 0
for(def d : dbcursor){
index++
if(index<5000){
continue
}
// rows << d
// if(rows.size()>=100){
// updateByEs(rows)
// rows.clear()
// }
updateByEs(d, index)
}
}
def updateByEs(row, index){
BulkRequest request = new BulkRequest();
def client = ElasticUtils89.getClient("192.168.0.89")
int processedCount = 0;
def id = row.get("_id").toString()
def doc = [:]
doc.put("province",row.province)
doc.put("short_province",row.short_province)
doc.put("city",row.city)
doc.put("short_city",row.short_city)
doc.put("district",row.district)
doc.put("short_district",row.short_district)
doc.put("name", row.name)
doc.put("location", row.location)
doc.put("bd_location", row.bd_location)
doc.put("shape", row.shape)
doc.put("bd_shape", row.bd_shape)
request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON));
if (request.numberOfActions() >= 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
// println "Processed " + processedCount + " documents"
}
}
}
}
@@ -5,21 +5,15 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.joda.time.Instant
import java.time.format.DateTimeFormatter
import java.util.regex.Pattern
class DevProcess {
@@ -64,6 +58,7 @@ class DevProcess {
// task.deleteAddr()
// task.deleteAddr2()
task.synchDevTo100()
// task.statShape()
// task.addNewDev2()
// task.addAlias()
@@ -77,7 +72,7 @@ class DevProcess {
// println DateUtils.add("2024-7-24", 7);
// task.addKfc()
// task.devToPark() //开发区下辖园区
// task.devToPark() //开发区下辖园
// task.checkTown()
// task.repeatSave()
@@ -87,6 +82,9 @@ class DevProcess {
// task.matchDevShape()//匹配开发区围栏
// task.addQiDevResult() //获取齐梁数据结果
// task.updateDevShape()//修改开发区结构
}
@@ -442,30 +440,33 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
if(!d['B'] && d['A']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
if(!d['E'] && d['D']){
def q = new BasicDBObject()
// q.put("name", d['A'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['A']]))
def name = d['A'].replace('经济技术开发区','经济开发区')
// q.put("name", d['D'])
q.put("city",d['C'])
// q.put('alias', new BasicDBObject('$in',[d['D']]))
def name = d['D'] //.replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['I'] = "库中有开发区-相似2"
d['G'] = "相似"
// d['K'] = obj.name
// d['C'] = obj._id.toString()
d['B'] = obj.name
// if(obj.name == d['A']){
// d['E'] = "相同"
d['E'] = obj.name
// if(obj.name == d['D']){
// d['G'] = "相同"
// }else{
// d['E'] = "包含"
// d['G'] = "包含"
// }
if(obj.shape){
d['F'] = "是"
}
// d['E'] = "别名"
// d['F'] = obj.name
}
}
},1);
},0);
}
def calAddrDistince(){
@@ -794,6 +795,18 @@ class DevProcess {
development_zone_CONN.insert(rets)
}
def statShape(){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("shape", new BasicDBObject('$exists', true)))
def index = 0
for(def r : dbcursor){
index ++
def coordinates = r.shape.coordinates
println index+" "+r.name+" "+coordinates.size()
}
}
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
@@ -1148,4 +1161,26 @@ class DevProcess {
}
},0)
}
def updateDevShape(){
def q = new BasicDBObject()
q.put("shape", new BasicDBObject('$exists', true))
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(q)
for(def d : dbcursor){
if(d.name!="福州长乐国际机场综合保税区"){
def udata = new BasicDBObject()
if(d.shape.coordinates){
d.shape.coordinates = [d.shape.coordinates]
udata.put("shape", d.shape)
}
if(d.bd_shape && d.bd_shape.coordinates){
d.bd_shape.coordinates = [d.bd_shape.coordinates]
udata.put("bd_shape", d.bd_shape)
}
def updateQuery = new BasicDBObject("_id",d._id)
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
}
@@ -0,0 +1,113 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.ExcelUtil
import com.fangjia.script.gj.ProxyHttpUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
class DevQlCheck {
//http://146.56.217.44:8088/uams/index
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static main(args) {
def task = new DevQlCheck()
task.run()
}
def run(){
def devName = "福建龙海经济开发区"
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",devName))
def pois = readCSV("C:\\Users\\gaojuan\\Downloads\\漳州市POI.csv")
def devPois = []
devPois << ["省", "城市", "区域", "名称", "地址", "类型", "经度", "纬度", "齐梁开发区"]
for(def poi : pois){
def flag = InsidePolygon.isInArea(poi.lng.toString().toDouble(), poi.lat.toString().toDouble(), InsidePolygon.createPoint4(obj.shape.coordinates))
if(flag){
devPois << [poi.province, poi.city, poi.district, poi.name, poi.address, poi.cat, poi.lng, poi.lat]
}
}
println "共:"+devPois.size()+" 个"
ExcelUtils.write("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx", devPois, "齐梁开发区")
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区外包\\${devName}poi齐梁校验.xlsx",{d->
def address = d['B']+d['C']+d['D']
// if(d['E']){
// address = address + d['E']
// }
Thread.sleep(3000)
def s = getQlResult(address)
d['I'] = s
index ++
println index+" "+ s
})
}
def getQlResult(address){
def params = [
'address': address,
'type': '1' //1:对公 0:对私
]
try{
def str = ProxyHttpUtils.getProxyResult(targetUrl, params);
println address +" "+ str
JSONObject result = JSONObject.fromObject(str)
if(result && result.code==20000){
def data = result.data
if(data){
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
return s
}
}
}
}catch(Exception e){
println address +" "+e.getMessage()
}
return null
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
try{
o.put("province", arrs[0])
o.put("city", arrs[1])
o.put("district", arrs[2])
o.put("name", arrs[3])
o.put("township", arrs[4])
o.put("address", arrs[5])
o.put("cat", arrs[6])
o.put("lng", arrs[7])
o.put("lat", arrs[8])
o.put("data_source", arrs[9])
rets << o
}catch(Exception e){
println filename+" 文件异常"
}
}
}
return rets;
}
}
@@ -7,7 +7,6 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -34,8 +33,8 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
task.addDev(path)
// task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -168,7 +167,7 @@ class DevUpdateTestBatch {
println d['A'] +" 不存在"
}
}
},2)
},1)
}
@@ -1,12 +1,14 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -28,14 +30,24 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
task.addOldMatch2()
// task.addOldMatch2()
// task.runDb()
// task.hainanMatch()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
// task.runMatch()
// task.checkDist()
task.addPrice()
}
def addPrice(){
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
}
def addOldMatch(){
@@ -94,6 +106,46 @@ class GXDdistrictMatch {
}
def hainanMatch(){
def fileName = "D:\\数据\\小区\\海南未匹配地址-1.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['D'] && d['F'] ){//
// def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['I'])
// if(jsonObject){
//// if(jsonObject.get("district") && jsonObject.get("district")!='null'){
//// d['AO'] = jsonObject.get("district")
//// }
// if(jsonObject.get("province") && jsonObject.get("province")!="null"){
// d['AP'] = jsonObject.get("province")
// }
// if(jsonObject.get("city") && jsonObject.get("city")!="null"){
// d['AI'] = jsonObject.get("city")
// }
// }
def formatCity = RegionData.formatCity(d['F'])
def fullCityObj = fullCityMap.get(formatCity)
if(fullCityObj){
// def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
def obj = findDatasByCityAndName(formatCity, d['D'].replaceAll('[·\\(\\)()]|·', ''))
if(obj){
d['U'] = "楼盘抓取库"
d['V'] = obj.city
d['W'] = obj.region
d['X'] = obj.name
d['Y'] = obj.property_type
d['Z'] = obj.address
def loc = obj.location
if(loc){
d['AA'] = loc.x
d['AB'] = loc.y
}
}
}
}
})
}
def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
@@ -238,43 +290,41 @@ class GXDdistrictMatch {
def saveToDB(){
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
def fileName = "D:\\数据\\小区\\海南新增小区.xlsx"
ExcelUtils.modify(fileName, { d ->
if(d['A']){
if(d['A'] && d['C']){
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def alias = []
if(d['G'] && name!=d['G']){
alias << d['G']
}
// def alias = []
// if(d['G'] && name!=d['G']){
// alias << d['G']
// }
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
// alias << d['AI']
// }
if(alias){
udata.put("alias", alias)
}
// if(alias){
// udata.put("alias", alias)
// }
def addr = AddressUtil.formatAddress(d['A'], d['D'])
def addr = AddressUtil.formatAddress(d['A'], d['E'])
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("status", 0)
udata.put("name_cat", 0)
udata.put("cat", "district")
udata.put("username", "高娟")
// udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
// if(d['AH']){
// type = d['AH']
// }else{
// type = d['Y']
// }
if(d['D']){
type = d['D']
}
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
@@ -282,26 +332,50 @@ class GXDdistrictMatch {
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
udata.put("estate_type2", estateType2)
if(d['F']){
Long lng = d['F']*10000000000
udata.put("lng", lng)
}
if(d['E']){
Long lat = d['E'] * 10000000000
Long lat = d['G'] * 10000000000
udata.put("lat", lat)
}
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
def opt = "add"
if(s && s=='成功'){
saveOptLog(udata.city, udata.region, udata.name, opt)
udata.put("lng", lng)
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(d['F'].toString().toDouble(), d['G'].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
if(lng2){
Long lng2_new = lng2 *10000000000L
Long lat2_new = lat2 *10000000000L
udata.put("lng2", lng2_new)
udata.put("lat2", lat2_new)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map",markMap)
// udata.put("is_new", 0)
DbUtils.seaweed.save(udata)
saveOptLog(udata.city, udata.region, udata.name, "add")
// def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println name
// def opt = "add"
// if(s && s=='成功'){
// saveOptLog(udata.city, udata.region, udata.name, opt)
// }
}
})
}
def checkDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\海南未匹配地址-1.xlsx", { d ->
if(d['V'] && d['W'] && d['X']){
matching(d['V'], d['W'], d, d['X'], null, null)
}
})
}
def match(city, region, d){
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
@@ -391,7 +465,7 @@ class GXDdistrictMatch {
if(rets && rets.size()>=1){
d = setRowValue(rets[0], d, "名称在地址中"+str)
}else{
if (d['D']) {
if (d['D'] && alias) {
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
if (rets) {
if (rets.size() >= 1) {
@@ -449,14 +523,14 @@ class GXDdistrictMatch {
def setRowValue(dbo, d, remark){
if (dbo) {
d['V'] = dbo.city
d['W'] = dbo.region
d['X'] = dbo.name
d['Y'] = dbo.estate_type2
d['Z'] = dbo.status
d['AA'] = dbo.name_cat
d['U'] = remark
d['AH'] = dbo.address
// d['V'] = dbo.city
// d['W'] = dbo.region
// d['X'] = dbo.name
// d['Y'] = dbo.estate_type2
d['AK'] = dbo.status
// d['AA'] = dbo.name_cat
d['AJ'] = remark
// d['AH'] = dbo.address
}
return d;
}
@@ -510,6 +584,33 @@ class GXDdistrictMatch {
return null;
}
def findDatasByCityAndName(def city, def name){
def sql = "select * from community where city='${city}' and name='${name}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
println d.property_type
})
if(rets){
def obj = null
for(def r : rets){
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
obj = r
break
}
}
if(obj){
return obj;
}else{
return rets[0];
}
}
return null;
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
@@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* Ãû¼¼¯
*/
@@ -31,12 +29,13 @@ class MinglujiData {
}
def run(){
def prov = "河北省"
def city = "石家庄市"
def prov = "½­ËÕÊ¡"
def city = "ÎÞÎý"
exportDatas(city)
// exportDatas(city)
// matchDev(prov, city)
// matchCommunity(city)
fiveLevelSplit(city)
// fiveLevelSplit(city)
// nineLevelSplit(city)
// tempMatchDev(prov)
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -39,12 +46,13 @@ class NewGaoDePoiBatch extends BaseBatch{
@Override
void process(Object args) {
run()
// run()
oldPoiMagre()
}
def run(){
if (CommonDatautils.isWindow()) {
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
TAGS = getAllPoiType("D:\\数据\\数据建设\\Poi\\POI分类白名单.xlsx")
}else{
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
}
@@ -145,6 +153,7 @@ class NewGaoDePoiBatch extends BaseBatch{
saveObj.data_id = map.map_id
saveObj.data_source = "¸ßµÂµØÍ¼"
saveObj.data_source_year = "2024-11"
saveObj.parent_id = map.parent_id
if(saveObj.cat && TAGS.contains(saveObj.cat)){
saveObj.status = 0
@@ -314,7 +323,7 @@ class NewGaoDePoiBatch extends BaseBatch{
rows.each{map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
request.add(new IndexRequest("map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
// ÅúÁ¿Ìá½»Êý¾Ý
if (request.numberOfActions() >= batchSize) {
@@ -346,4 +355,177 @@ class NewGaoDePoiBatch extends BaseBatch{
}
def oldPoiMagre(){
def cityList = CommonDatautils.findFullNameCitys().reverse()
def index = 0
for(def cityObj : cityList){
index ++
if(index <291){
continue
}
def fileName = "/home/goojia/gj/poi/old_poi/merge_poi/${cityObj.city}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
}
println fileName
File file=new File(fileName);
if(file.exists()){
def results = readCSV(fileName)
println "运行城市:"+cityObj.province+" " +cityObj.city+" "+results.size()
if(results){
def megerMap = poiRepeat(cityObj.city, results)
println "待保存:" + cityObj.city +" "+megerMap.size()
save(megerMap.values())
}
}else{
println fileName +" 不存在"
}
}
}
def poiRepeat(city, oldPois){
def rets = new ConcurrentHashMap()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(oldPois.size())
def townMap = CommonDatautils.queryTownByCity(city)
def esMap = findEsPoiByCity(city)
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
for(def i=0;i<oldPois.size();i++){
final def p = oldPois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def nameMap = esMap.get(p.district)
def nameKeys = nameMap.keySet()
def key = p.name.replace("-","").replace("·","")
if(!nameKeys.contains(key)){
def simlarObj = null
if(esMap){
simlarObj = findSimlarPoi(p, esMap)
}
if(!simlarObj){
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
if(!rets.containsKey(newKey)){
p.data_source_year = "2023-08"
rets.put(newKey, p)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findSimlarPoi(ret, esMap){
def simlarObj = null
def region = ""
if(ret.district){
region = ret.district
}
def regionMap = esMap.get(region)
if(regionMap){
def keys = regionMap.keySet()
for(def key : keys){
def esValue = regionMap.get(key)
if(esValue.township && ret.township){
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
if(ret.location && ret.location.lon){
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
if(distance <100){
simlarObj = esValue
println esValue.name +" "+ret.name+" "+distance
break;
}
}else{
println ret.data_id +" 无经纬度信息"
}
}
}
}
}else{
println "无数据"
}
return simlarObj
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def map = new HashMap()
ElasticUtils89.executeQuery("192.168.0.89", "map_poi_2024", queryBuilder, { String id, Map d->
def district = "嘉峪关市"
if(d.district){
district = d.district
}
def regionMap = map.get(district)
if(!regionMap){
regionMap = new HashMap()
}
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
regionMap.put(d.name.replace("-","").replace("·",""), o)
// map.put(district, regionMap)
map.put(district, regionMap)
})
return map
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("province",arrs[0])
o.put("city" , arrs[1])
o.put("district",arrs[2])
o.put("name",arrs[3])
o.put("township",arrs[4])
o.put("address",arrs[5])
o.put("cat",arrs[6])
o.put("type",arrs[7])
o.put("secondary_type",arrs[8])
o.put("small_type" ,arrs[9])
if(arrs[10]){
o.put("data_id",arrs[10])
}else{
o.put("data_id",arrs[11])
}
o.put("spell_name",arrs[12])
def location = [:]
if(arrs[13]){
location.lon = arrs[13].toString().toDouble()
location.lat = arrs[14].toString().toDouble()
}
o.put("location",location)
o.put("data_source",arrs[15])
o.put("status",arrs[16])
o.put("version",arrs[17])
rets << o
}
}
return rets;
}
}
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class NewPoiData {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new NewPoiData()
task.run()
}
def run(){
// export()
// exportNetWorkOffice()
// matchByCity()
// noMatchData()
calcuteDistance()
}
def noMatchData(){
ExcelUtils.multiThreadingModify("d:\\数据\\未匹配写字楼2.xlsx", { d ->
if(d['H']!='地图'){
def map = CommonDatautils.findOneThirdNetWorkOffice(d['A'], d['B'], d['C'])
if(map){
d['E'] = map.lng
d['F'] = map.lat
}
}
})
}
def matchByCity(){
ExcelUtils.multiThreadingModify("d:\\数据\\入库写字楼.xlsx",{d->
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['A']+d['B']+d['C'], "office", false)
def formatAddress = null
if(d['D']){
formatAddress = AddressUtil.formatAddress(d['A'], d['D'])
d['G'] = formatAddress
}
if(!jsonObject && formatAddress){
if(DataUtil.isPreciseAddress(formatAddress)){
def address = d['A']
if(d['B']){
address = address + d['B']
}
address = address + formatAddress
jsonObject = FangjiaClientUtils.newMatch(d['A'], address, "office", false)
}
}
if(jsonObject) {
d['H'] = jsonObject.get("id")
d['I'] = jsonObject.get("city")
d['J'] = jsonObject.get("district")
d['K'] = jsonObject.get("name")
d['L'] = jsonObject.get("propertyType")
d['M'] = jsonObject.get("credit")
}
})
}
def export(){
def rets = []
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
rets.addAll(CommonDatautils.findNewPoiDatas())
ExcelUtils.write("d:\\数据\\poi写字楼.xlsx", rets, "写字楼")
}
def exportNetWorkOffice(){
def rets = []
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
rets.addAll(CommonDatautils.findThirdNetWorkOffice())
ExcelUtils.write("d:\\数据\\poi写字楼-网站.xlsx", rets, "写字楼")
}
def calcuteDistance(){
ExcelUtils.multiThreadingModify("d:\\数据\\写字楼测试-1118-新增.xlsx", { d ->
if(d['F']){
if(d['A'].contains("北京")){
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 39.996743, 116.481075)
d['K'] = distance
}else if(d['A'].contains("烟台")){
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 37.459645, 121.438616)
d['K'] = distance
}
}
}, 1)
}
def addDistrict(){
def path = "D:\\数据\\入库写字楼.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def city = d['A']
def region = d['B']
def name = d['C']
def formatAddress = null
def address = d['D']
if(address){
if(d['G']){
formatAddress = AddressUtil.formatAddress(city, address)
}
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
def key = city+"##"+region+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","office")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
}
}
},0)
}
}
@@ -1,22 +1,20 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
@BatchOwner(Watcher.GaoJuan)
class PoiNineSplitBatch extends BaseBatch{
class PoiNineSplitBatch{
static main(args) {
runBatch(new PoiNineSplitBatch(), args)
public static void main(args) {
def task = new PoiNineSplitBatch()
task.run()
}
@Override
void process(Object args) {
run()
}
// @Override
// void process(Object args) {
// run()
// }
def run(){
def i = 0;
while(i<360) {
@@ -4,9 +4,12 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -31,12 +34,16 @@ class RegionStandardBatch extends BaseBatch{
@Override
void process(Object args) {
seaweedRegionStandard()
// poiRegionStandard()
// seaweedRegionStandard()
poiRegionStandard()
// poiRegionStandard()
//通过乡镇街道修改区域 如:东莞市
// updateRegionByTown()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("district")
seaweedRegionStandardByCat("office")
}
def poiRegionStandard(){
@@ -93,7 +100,7 @@ class RegionStandardBatch extends BaseBatch{
def seaweedRegionStandardByCat(cat){
def citys = CommonDatautils.findAllCity()
def citys = ["伊春"]//CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
@@ -148,8 +155,12 @@ class RegionStandardBatch extends BaseBatch{
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
println city +" "+ districts.size()
def index = 0
for(def d : districts){
index ++
println index +" "+ d.region
if(set.contains(d.region)){
continue
}
@@ -157,8 +168,12 @@ class RegionStandardBatch extends BaseBatch{
def updateRegion = null
def flag = null
for(def key : set){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def obj = streetMap.get(key)
def alias = null
if(obj && obj.alias){
alias = obj.alias
}
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
@@ -166,13 +181,26 @@ class RegionStandardBatch extends BaseBatch{
flag = "pass"
break;
}
if(obj.shape){
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
if(inAreaFlag){
def newRegion = obj.street
newRegion = RegionData.formatRegion(d.city, newRegion)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" ÇøÓòÐÞ¸ÄΪ£º"+updateRegion
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
@@ -402,4 +430,27 @@ class RegionStandardBatch extends BaseBatch{
return rets;
}
def updateRegionByTown(){
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
def town = RegionData.formatRegion(d['B'], d['F'])
if(town){
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
if(!obj){
def udata = new BasicDBObject()
udata.put("region", town)
udata.put("m_date", new Date())
println d['F']+" "+" 区域修改为:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("status", 1)
println d['F']+" "+" 删除:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
})
}
}
@@ -16,7 +16,7 @@ class RegionUpdateBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionUpdateBatch()
@@ -0,0 +1,110 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
class SynchAdministrativeMappingDataBatch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
runBatch(new SynchAdministrativeMappingDataBatch(), args);
}
@Override
public void process(Object args) {
def indexName = "administrative_mapping"
def retMap = synchAdminiMappingTo100()
synchToEs(retMap, "192.168.0.89", indexName)
synchToEs(retMap, "192.168.0.52", indexName)
println(" 运行完成")
}
def synchAdminiMappingTo100(city){
//删除旧的,再同步新的
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000)
def datas = []
def retMap = [:]
for(def d : dbcursor){
datas << d
retMap.put(d._id.toString(), d)
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
datas.clear();
}
return retMap;
}
def synchToEs(retMap, host, indexName){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
def oldIds = new CopyOnWriteArrayList()
ElasticUtils89.queryUpdate(host, indexName, queryBuilder, { String id, doc->
oldIds << id
def mongoObj = retMap.get(id)
def updateFieldCount = 0
mongoObj.each{k,v->
if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){
def val = doc.get(k)
if(val!=v){
if(k!="shape" || (k=="shape" && v.type!="MultiPolygon")){
println k +" 修改 "+id
updateFieldCount++
doc.put(k,v)
}
}
}
}
if(mongoObj && updateFieldCount){
//进行修改
return true
}else{
if(!mongoObj){
println "删除:"+id +" "+doc.name
ElasticUtils.delete(id, host, indexName)
return false
}
}
})
def addEsDatasMap = [:]
retMap.each{id,map->
if(!oldIds.contains(id)){
println "需新增:"+id
map.remove("c_date")
map.remove("m_date")
map.remove("polyline_source")
map.remove("operator")
map.remove("_id")
addEsDatasMap.put(id, map)
}
}
if(addEsDatasMap){
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
}
}
}
@@ -0,0 +1,142 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
/**
* 将43上的开发区同步到mongo-100, es-89, es-52
*/
class SynchDevDataBatch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static main(args) {
runBatch(new SynchDevDataBatch(), args);
}
@Override
public void process(Object args) {
def indexName = "development_zone"
def devMap = synchDevTo100()
synchDevEs(devMap, "192.168.0.89", indexName)
synchDevEs(devMap, "192.168.0.52", indexName)
}
def synchDevTo100(){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
def rets = []
def devMap = [:]
for(def r : dbcursor){
rets << r
devMap.put(r._id.toString(), r)
}
//删除旧的,再同步新的
development_zone_CONN.remove(new BasicDBObject())
development_zone_CONN.insert(rets)
return devMap;
}
def synchDevToEs(devMap){
// def esDevs = findEsDevDatas()
// def ids = []
// for(def dev : devDatas){
// ids << dev._id.toString()
// def esObj = esDevs.get(dev._id.toString())
// def updateValMap = [:]
// if(esObj){
// dev.each{k,v->
// if(k!="id" && k!="_id"){
// def val = dev.get(k)
// if(val!=v){
// updateValMap.put(k,val)
// }
// }
// }
// }
// if(!esObj || (esObj && updateValMap)){
// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.89", "development_zone")
// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.52", "development_zone")
// }
// }
//
// def esDelIds = []
// esDevs.each{id,map->
// if(!ids.contains(id)){
// ElasticUtils.delete(id, "192.168.0.89", "development_zone")
// ElasticUtils.delete(id, "192.168.0.52", "development_zone")
// esDelIds << id
// }
// }
// println "Es删除开发区数量:"+esDelIds.size()
}
def synchDevEs(devMap, host, indexName){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
def oldIds = new CopyOnWriteArrayList()
ElasticUtils89.executeQuery(host, indexName, queryBuilder, { String id, doc->
oldIds << id
def mongoObj = devMap.get(id)
def updateFieldCount = 0
mongoObj.each{k,v->
if(!["id","_id" ,"build_date" ,"create_time","bd_shape","img_url","operator","operate_result","m_date","township","block"].contains(k)){
def val = doc.get(k)
if(val!=v){
println doc.name +" 修改 "+k+" "+ v+" "+id
updateFieldCount++
doc.put(k,v)
}
}
}
if(mongoObj && updateFieldCount){
//进行修改
return true
}else{
if(!mongoObj){
println "删除:"+id +" "+doc.name
ElasticUtils.delete(id, host, indexName)
return false
}
}
})
def addEsDatasMap = [:]
devMap.each{id,map->
if(!oldIds.contains(id)){
println "需新增:"+id
map.remove("create_time")
map.remove("build_date")
map.remove("bd_shape")
map.remove("img_url")
map.remove("operator")
map.remove("operate_result")
map.remove("m_date")
map.remove("township")
map.remove("block")
map.remove("_id")
addEsDatasMap.put(id, map)
}
}
if(addEsDatasMap){
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
}
}
}
@@ -1,7 +1,13 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -9,17 +15,79 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class TempExportData {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection qianzhan_coll = MongoUtils.getCollectionByName("crawl", "yuanqu_qianzhan_com_map", "192.168.0.206", 27020)
static main(args) {
def task = new TempExportData()
// task.run()
def rets = task.fzHouseStat()
ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
println "文件已生成"
// def rets = task.fzHouseStat()
// ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
// println "文件已生成"
// exportDistShape()
// task.exportDistrictAlias()
task.exportOfficeHouseCount()
}
def exportDistShape(){
def task = new BasicDBObject()
}
def exportDistrictAlias(){
def citys = CommonDatautils.findAllCity()
def allRets = []
allRets << ["Id","城市","区域","名称","地址","已删ID","已删区域","已删名称","已删地址","距离"]
for (def city : citys) {
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
def rets = findDelDistBatch(dists)
allRets.addAll(rets)
}
ExcelUtils.write("D:\\数据\\数据建设\\小区\\融合存疑小区.xlsx",allRets, "小区")
}
def findDelDistBatch(dists){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def nameAlias = dist.alias
if(nameAlias){
for(def nameAlia : nameAlias){
def delDists = CommonDatautils.findAllStatusDistrict(dist.city, dist.region, nameAlia, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
if(delDists){
if(delDists.size()==1){
def delDist = delDists[0]
def distance = "缺经纬度"
if(delDist.lat){
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, delDist.lat/10000000000.0, delDist.lng/10000000000.0)
}
println totalCount.get()+" "+dists.size() +" "+dist.city
rets << [dist._id.toString(), dist.city, dist.region, dist.name, dist.address,delDist._id.toString(), delDist.region, delDist.name, delDist.address, distance]
}
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def run() {
@@ -33,6 +101,8 @@ class TempExportData {
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
}
def getResults(city) {
def rets = []
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
@@ -120,4 +190,75 @@ class TempExportData {
return rets
}
def findHouseCountByCat(def city, def region, def name, def cat){
String sql = "select count(*) as c from seawater where cat='${cat}' and city='${city}' and district_name='${name}' "
if(region){
sql = sql + " and region='${region}'"
}
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}
def findYuanQu(city, name){
def q = new BasicDBObject("name", Pattern.compile("^${name}"))
def dbcursor = qianzhan_coll.find(q)
def obj = null
for(def d : dbcursor){
if(d.city.contains(city)){
obj = d
break;
}
}
return obj
}
def exportOfficeHouseCount(){
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['T'] = "前瞻存在"
// }
// if(!d['T']){
// def city = RegionData.formatCity(d['C'])
// def obj = findYuanQu(city, d['E'])
// if(obj){
// d['T'] = "前瞻存在"
// d['U'] = obj.name
// }
// }
// if(!d['T']) {
// def city = RegionData.formatCity(d['C'])
// def region = RegionData.formatRegion(city, d['D'])
//
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
// if (!count) {
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
// }
// d['T'] = count
// }
if(!d['T']){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
if(datas){
def sources = []
for(def o : datas){
sources << o.source
}
if(sources){
d['V'] = sources.join(",")
}
}
}
})
}
}
@@ -1,105 +1,57 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
}
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
}
import org.bson.types.ObjectId
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
Date creationDate = objectId.getDate();
System.out.println(creationDate);
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
//
//
//def city = "昌江"
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
// def date = d['B']
// def start = DateUtils.getFirstDayOfMonth(date)
// def end = DateUtils.getLastDayOfMonth(date)
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
// for(def weekend : weekends){
// def avg_price = d['I']
// def val = random(-200,200)
// def new_price = (int)(avg_price*val/10000 + avg_price)
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
//
// def o = new BasicDBObject()
// o.put("avg_price", new_price)
// o.put("c_date" , new Date())
// o.put("city", city)
// o.put("m_date" , new Date())
// o.put("weekend", weekend)
// o.put("cat","城市")
// o.put("moving_average" , (int)avg_price)
// if(!last_price){
// o.put("weekend_markup" , 0)
// }else{
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
// o.put("weekend_markup" , weekend_markup)
// }
//
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
// }
//},1)
//
//
//public static final int random(int i1, int i2) {
// int max = Math.max(i1, i2);
// int min = Math.min(i1, i2);
// return min + RandomUtils.nextInt(max - min + 1);
//}
//
//
//
//
//
@@ -52,9 +52,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
def index = 0
for(def c : citys){
index ++
if(index < 206){
continue
}
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
matchByCity(c, noMatchDatas)
@@ -123,9 +120,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
def index = 0
for(def c : citys){
index ++
if(index < 70){
continue
}
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
def addCount = addDatasToDB(crawlDatas)
println c + " ÐÂÔö£º"+ addCount
@@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -45,6 +44,8 @@ class TownMiddleProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
@@ -64,7 +65,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
// synchTown()
synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -95,7 +96,13 @@ class TownMiddleProcess {
// addTownCenter()//百度接口添加中心点
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
addSpecialCityTownAlias()
// addSpecialCityTownAlias()
// updateNewRegionPoyline()//更新新的区域围栏
updateMappingPolyline()
}
@@ -451,16 +458,16 @@ class TownMiddleProcess {
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
}
@@ -681,7 +688,7 @@ class TownMiddleProcess {
}
def regionPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
@@ -1356,4 +1363,90 @@ class TownMiddleProcess {
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
}
def updateNewRegionPoyline(){
def regionPolylineMap = regionPolyline()
def q = new BasicDBObject()
q.put("type","区域")
// q.put("city", "苏州市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
for(def d : dbcursor){
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
println d.city+" "+d.district
if(shapeData){
println "新围栏数:"+shapeData.shape.coordinates.size()
}
if(d.shape){
println "旧围栏数:"+d.shape.coordinates.size()
}
if(d.shape && shapeData && shapeData.shape!=d.shape){
def udata = new BasicDBObject()
udata.shape = shapeData.shape
udata.location = shapeData.location
udata.m_date = new Date()
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
// if(Math.abs(area1-area)>=1){
// println "新:"+area
// println "旧:"+area1
// def udata = new BasicDBObject()
// udata.shape = shapeData.shape
// udata.location = shapeData.location
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// }
// }
}
}
}
def updateMappingPolyline(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("shape.type","polygon")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
updateAdministrativeMapping(d._id.toString(), d.shape)
}
}
def updateAdministrativeMapping(def id, def shape){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.shape = params.newshape",
"params":{
"newshape":{
"type":"${shape.type}",
"coordinates":${shape.coordinates}
}
}
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -1,17 +1,36 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -22,6 +41,7 @@ class TownStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def updateList = []
def addAliasList = []
@@ -37,12 +57,17 @@ class TownStandardBatch extends BaseBatch{
}
def TownStandard(){
// townStandardByCat("office")
townStandardByPoi()
townStandardByCat("office")
// townStandardByPoi()
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// standardTown()
// updateTownData()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
def citys = ["天津"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -53,10 +78,10 @@ class TownStandardBatch extends BaseBatch{
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias","shape"], mzbRegionMap)
if(townMap){
def regionSet = townMap.keySet()
if(regionSet.size()>1){
if(regionSet.size()>=1){
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
}else{
println city +" 无区域"
@@ -69,11 +94,15 @@ class TownStandardBatch extends BaseBatch{
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
//浙江,江苏,安徽
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
def runProvinces = ["福建省","浙江省","江苏省","安徽省","山东省","甘肃省"]
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index++
def data = fullCityMap.get(city)
// if(runProvinces.contains(data.province)){
// continue
// }
def fullCityName = null
if(data){
fullCityName = data.city
@@ -95,11 +124,16 @@ class TownStandardBatch extends BaseBatch{
def rets = readPoiCsv(fileName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data)
}else{
// haveStreetProcess(city, set, cat)
}
}
if(updateList.size()>50000){
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-${index}.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名-${index}.xlsx", addAliasList, "数据")
updateList.clear()
}
}else{
println fileName+" 文件没找到"
}
@@ -107,18 +141,18 @@ class TownStandardBatch extends BaseBatch{
println "没找到相应的全称:"+city
}
}
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-最后.xlsx", updateList, "数据")
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
if(fullCityName){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township","lng","lat"])
for(def d : districts){
if(d.township){
def towns = null
def updateTown = null
def towns = null
def updateTown = null
if(d.region){
for(def m : townMap){
def fullRegionName = shortRegionNameMap.get(d.region)
if(m.key == fullRegionName){
@@ -126,41 +160,51 @@ class TownStandardBatch extends BaseBatch{
break
}
}
}
if(!towns){
towns = townMap.get(fullCityName)
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
if(town.shape){
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
updateTown = town.street
break
}
}
}
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
@@ -168,7 +212,7 @@ class TownStandardBatch extends BaseBatch{
}
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap, data){
def results = []
if(fullCityName){
for(def d : rets){
@@ -221,21 +265,21 @@ class TownStandardBatch extends BaseBatch{
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.province, d.city, d.district, d.name,d.township, d.status, updateTown]
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
// results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
results << m
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.province, d.city, d.district, d.name, d.township, d.status, ""]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
}
}
@@ -244,10 +288,10 @@ class TownStandardBatch extends BaseBatch{
}
}
// if(results.size()>0){
// updateByEs(results)
// results.clear()
// }
if(results.size()>0){
updateByEs(results)
results.clear()
}
println fullCityName +" 处理完成"
}
@@ -322,18 +366,475 @@ class TownStandardBatch extends BaseBatch{
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
try{
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}catch(Exception e){
println arrs
}
}
}
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=11 && arrs[10]=="是" && arrs[6]!=arrs[7]){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
o.put("city", arrs[2])
o.put("old_district", arrs[3])
o.put("new_district", arrs[8])
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def readCSV2(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
def o = new BasicDBObject()
try{
o.put("id", arrs[1])
o.put("province", arrs[2])
o.put("city", arrs[3])
o.put("district", arrs[4])
o.put("lng", arrs[5])
o.put("lat", arrs[6])
o.put("data_source", arrs[7])
if(arrs.size()>=9){
o.put("township", arrs[8])
o.put("new_district", arrs[9])
o.put("town_source", arrs[10])
}
rets << o
}catch(Exception e){
println arrs
}
}
index++
}
return rets;
}
def addTownByTownShape(){
def i = 0
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
while(i<=50){
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\no_township_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
def rets = readCSV2(fileName)
println i +" "+ rets.size()
def rows = addTownshipBatch(rets)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
println fileName +" 运行完成"
}else{
println fileName +" 不存在"
}
i++
}
}
def standardTown(){
def i = 49
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
def cityTownMap = CommonDatautils.queryAllTown(["city", "district", "mzb_district_name", "street", "alias"])
while(i<=50){
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
def rets = readCSV2(fileName)
println i +" "+ rets.size()
def rows = addTownshipBatch2(rets, cityTownMap)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
println fileName +" 运行完成"
}else{
println fileName +" 不存在"
}
i++
}
}
def addTownshipBatch(def rets){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
if(!rMap){
def data = fullCityMap.get(r.city)
rMap = regionFullNameMap.get(data.city)
}
if (rMap) {
// def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
// def set = rMap.keySet()
println totalCount.get() +" "+r.city + " "
// if (set.size() >= 1) {
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
// }
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return results
}
def addTownshipBatch2(def rets, def cityTownMap){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
def fullcity = null
if(!rMap){
def data = fullCityMap.get(r.city)
rMap = regionFullNameMap.get(data.city)
fullcity = data.city
}
if (rMap) {
def townMap = cityTownMap.get(fullcity)
def set = rMap.keySet()
if (set.size() >= 1) {
if(townMap.size()>=1){
r.strand_town = isStandTown(townMap, r.new_district, r.township)
}
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return results
}
def addTown(saveObj, townMap, location) {
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
// def towns = townMap.get(saveObj.district)
// if (towns) {
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.town_source = "同区域"
// saveObj.strand_town = "是"
// break
// }
// }
// }
// }
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
// if(!saveObj.township){
// def townNames = townMap.keySet()
// for(def townName : townNames){
// towns = townMap.get(townName)
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.new_district = town.mzb_district_name
// saveObj.town_source = "忽略区域"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// break
// }
// }
// }
// if(saveObj.township){
// break;
// }
// }
// }
//通过调取地图api接口获取乡镇街道
if(!saveObj.township){
def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
if(gaodeMap && gaodeMap.township){
saveObj.township = gaodeMap.township
saveObj.new_district = gaodeMap.district
saveObj.town_source = "高德API"
saveObj.strand_town = ""
// if(townMap.size()>=1){
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// }
}
}
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
// if(addressComponent.town){
// saveObj.township = addressComponent.town
// saveObj.new_district = addressComponent.district
// saveObj.town_source = "百度API"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// }
// }
// }
// }
return saveObj
}
def isStandTown(townMap, district, town){
def towns = townMap.get(district)
def flag = "非"
if(towns){
for(def t : towns){
if(t.street == town){
flag = "是"
break
}
}
}
return flag
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def stop = false
def count = 0
def jsonObject = null
// while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
city = addressComponent.city
format_addr = jsonObject.regeocode.formatted_address
}
}
}
// if(jsonObject.status==0 || count>3){
// stop = true
// println "高德调用超3次"
// }else{
// count ++
// Thread.sleep(3000)
// print "."
// }
// }
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
def updateTownData(){
def i = 49
while(i<50) {
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv"
}
File file = new File(fileName);
if (file.exists()) {
def rets = readCSV3(fileName)
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
if(rets){
// updateToEs(rets)//不能用这种方法修改,否则其它字段都将丢失
// updateToEs2(rets)
updateDistTownship(rets)
}
}
i++
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.containsKey("way")){
map.remove("way")
}
request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
updateMapPoi(map.id, map.township)
}
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/map_poi_2024/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
def updateDistTownship(rets){
for(def m : rets){
def obj = CommonDatautils.findDistrictById(m.id)
if(obj.township != m.township){
def udata = new BasicDBObject()
udata.put("township", m.township)
if(!m.new_district.contains(m.old_district)){
def lists = CommonDatautils.findAllDistrictByName(obj.city, null, obj.name, obj.cat)
if(lists.size()<2){
continue
}
def region = RegionData.formatRegion(m.city, m.new_district)
if(region){
def dbo = CommonDatautils.findDistrict(m.city, region, obj.name, obj.cat)
if(dbo){
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
println "删除 "+m.city +" "+obj.name+" "+m.id+" "+obj.cat
continue
}else{
udata.put("region", region)
}
}
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -11,6 +12,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -42,13 +44,16 @@ class TownshipStrandUpdate {
// task.updateTown()
// task.stat()
task.updateDiffRegionTown()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// println new Date().getTime()
// task.updateSpecCityTown()
task.updateTownshipAlias()
}
@@ -629,4 +634,69 @@ class TownshipStrandUpdate {
}
}
def updateTownshipAlias(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\20250219--乡镇街道添加别名.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject("type","乡镇街道").append("city", d["A"]).append("district",d["B"]).append("street",d["C"])
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
def newAlias = d['D']
if(list.size()==0){
list = q.put("street", d['D'])
newAlias = d['C']
list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
}
if(list && list.size()==1){
def obj = list[0]
def alias = obj.alias
if(!alias){
alias = []
}
if(!alias.contains(newAlias)){
alias << newAlias
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date", new Date())
udata.put("operator","陈杰")
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
// updateStreetAlias(obj._id.toString(), alias.join("','"))
}else{
println d["C"] +" "+ d["D"]
}
})
}
def updateStreetAlias(def id, def alias){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.alias=['${alias}']",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -0,0 +1,148 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class VillaHouseBatch extends BaseBatch{
def pgsqlInstance236 = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
//235为房价网内部计算均价的库
def sqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
runBatch(new VillaHouseBatch(), args)
}
@Override
void process(Object args) {
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city", "region", "name", "estate_type2"], "all")
def villaDists = findVillaDists(dists)
addVillaTypeFlag(villaDists)
println "运行完成"
}
def findVillaDists(dists){
def villaDists = []
for(def d : dists){
if(d.estate_type2 == "别墅"){
villaDists << d
}
}
return villaDists
}
def addVillaTypeFlag(def villaDists){
for(def i=0;i<villaDists.size();i++){
def dist = villaDists[i]
def sellIds = findHouseIdByCommId(dist._id.toString(), "310000")
println dist._id.toString() +" "+sellIds.size()
if(sellIds){
def retMap = run(sellIds)
if(retMap){
println "最终修改:"+retMap.size()
updateHouseFlag(retMap)
}
}
}
}
def run(sellIds){
def retMap = new ConcurrentHashMap()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(sellIds.size())
for(def i=0;i<sellIds.size();i++){
final def houseId = sellIds[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
retMap = findHouseInfo(houseId, retMap)
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return retMap
}
def findHouseIdByCommId(commid, cityId){
def sql = "select gujia_sell_id from base.bigdata_forsale where urbn_id='${cityId}' and gujia_comm_id='${commid}'";
def sellIds = []
sqlInstance235.eachRow(sql,{r->
sellIds << r.gujia_sell_id
})
return sellIds
}
def findHouseInfo(sell_id, houseMap){
def flag = findHouseDetail(sell_id)
if(flag){
houseMap.put(sell_id, flag)
}
return houseMap
}
def findHouseDetail(houseId){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", houseId))
def flag = null
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def house = houses[k]
def title = house.name
println title
if(title){
if(title.contains("独栋")){
flag = "独栋"
}else if(title.contains("联排")){
flag = "联排"
}else if(title.contains("叠拼")){
flag = "叠拼"
}else if(title.contains("花园洋房")){
flag = "花园洋房"
}else if(title.contains("双拼")){
flag = "双拼"
}
}
}
}
return flag;
}
def updateHouseFlag(retMap){
retMap.each{sell_id,type->
def sql = "update base.bigdata_forsale set villa_type='${type}' where gujia_sell_id='${sell_id}'"
sqlInstance235.executeUpdate(sql)
}
}
}
@@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.text.SimpleDateFormat
import java.util.concurrent.CopyOnWriteArrayList
@@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
@@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
final static def EMAILS = [
"gaojuan@fangjia.com"
@@ -68,6 +71,38 @@ class DistrictFieldAddBatch extends BaseBatch{
static main(args) {
runBatch(new DistrictFieldAddBatch(), args)
// ExcelUtils.multiThreadingModify("D:\\数据\\小区\\上海别墅.xlsx",{d->
// def dist = CommonDatautils.findDistrict(d['B'], d['G'], d['C'], "district")
//// if(dist){
//// d['L'] = dist.loudong_count
//// d['M'] = dist.household_count
//// d['Q'] = dist.estate_type2
//// }
// def list = findEsDatasByCityAndName(d['B'], d['G'], d['C'], "搜房")
// if(!list && dist){
// def alias = dist.alias
// for(def name : alias){
// list = findEsDatasByCityAndName(d['B'], d['G'], name, "搜房")
// if(list){
// d['Z'] = "别名"
// }
// }
// }else{
// d['Z'] = "名称"
// }
// if(list){
// def obj = list[0]
// if(obj.loudong_cnt){
// d['W'] = obj.loudong_cnt
// }
//
// if(obj.households){
// d['X'] = obj.households
// }
// d['Y'] = obj.url
// }
//
// })
}
@Override
@@ -83,18 +118,13 @@ class DistrictFieldAddBatch extends BaseBatch{
headers << fieldsNameMap.get(it)
}
headers << "抓取量"
headers << "匹配量"
headers << "0.9分以上量"
headers << "匹配率"
headers << "0.9以上匹配率"
// analysRets << headers
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
if(totalCount){
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表
}
}
println analysRets.toString()
@@ -107,8 +137,10 @@ class DistrictFieldAddBatch extends BaseBatch{
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
def matchRets1 = matchApi(esRets)
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
@@ -140,6 +172,65 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def addFieldFromMatchMapping(c, totalCount, analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findMatchSameDatas(c, esWebSource)
println esWebSource +" 新老算法匹配相同量:"+esRets.size()
def updateRets = processFieldsFromMatch(c , esRets, esWebSource)
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analysRets << analyRet
}
return analysRets;
}
def findMatchSameDatas(c, esWebSource){
def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true))
def siteDatas = thirdSeaweedMappingColl.find(query).toList()
def rows = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(siteDatas.size())
for(def i=0;i<siteDatas.size();i++){
final def d = siteDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def obj = findCommunities(d.url)
if(obj){
obj.match_id = d.match_id
rows << obj
}
// println d.city +" "+totalCount.get() +"/"+siteDatas.size()
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rows
}
def addFieldFromRonghe(c, totalCount, rh_analysRets){
@@ -209,14 +300,10 @@ class DistrictFieldAddBatch extends BaseBatch{
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
// continue
// }
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
@@ -224,7 +311,6 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
@@ -244,7 +330,9 @@ class DistrictFieldAddBatch extends BaseBatch{
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
if(newVal) {
newVal = newVal.toString()
}
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
@@ -312,12 +400,16 @@ class DistrictFieldAddBatch extends BaseBatch{
newValues.put(seaweedfield, newVal)
}
if(seaweedfield == "lng"){
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
try{
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
}
}catch(Exception e){
println esVal
}
}
}
@@ -361,6 +453,15 @@ class DistrictFieldAddBatch extends BaseBatch{
return list
}
static def findEsDatasByCityAndName(c, district, name, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' and region='${district}' and name='${name}'", {r->
list << r
})
return list
}
def fieldUpdate(total,updateRets){
def countMap = [:]
@@ -621,4 +722,26 @@ class DistrictFieldAddBatch extends BaseBatch{
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
}
}
def findCommunities(url){
if(!url){
return null
}
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
def rets = []
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
}catch(Exception e){
findCommunities(url)
}
def obj = null
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
obj = d
}
}
return obj;
}
}
@@ -0,0 +1,54 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class DistrictStatRentCleanBatch extends BaseBatch{
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
static main(args) {
runBatch(new DistrictStatRentCleanBatch(), args)
}
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def keys = CommonDatautils.getSimpleDistricts("district", c)
running(keys, c)
println c +" ÔËÐÐÍê³É"
}
}
def running(keys, c){
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
def fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("district_name", 1)
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
def delRets = []
for(def d : dbcursor){
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
println d.city+"@"+d.region+"@"+d.district_name
delRets << d._id
}
if(delRets.size()>=100){
def delquery = new BasicDBObject()
delquery.put("_id", new BasicDBObject('$in', delRets))
RENT_DIST_STAT_COLL.remove(delquery)
delRets.clear()
}
}
}
}
@@ -1,16 +1,9 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.HttpRequestUtil
import com.mongodb.DBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -336,7 +329,7 @@ class FangjiaClientUtils {
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=true&addressAssessment=false"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -354,7 +347,7 @@ class FangjiaClientUtils {
def jsonObj = null
def content = null
def url = null
address = address.replace("{","")
address = address.toString().replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
@@ -398,4 +391,32 @@ class FangjiaClientUtils {
return jsonObj;
}
/**
* 经纬度转换
* 1、高德转百度 2、高德转WGS84 3、百度转高德 4、WGS84转高德 5、百度转WGS84 6、WGS84转百度
*/
public static def convert(String locationstr, type){
def jsonObj = null
def content = null
def points = []
try {
def url = "http://192.168.0.19:31080/FangJiaNlpService/convert?locations="+URLEncoder.encode(locationstr,"UTF-8")+"&coordsys=${type}"
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
if(jsonObj && jsonObj.locations){
def locations = jsonObj.locations
for(def s : locations){
def arrs = s.split(",")
points = [arrs[0].toString().toDouble(), arrs[1].toString().toDouble()]
}
}
return points
} catch (Exception e) {
println e.getMessage()
}
return null;
}
}
@@ -0,0 +1,320 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.ExcelUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 写字楼字段填充入库(自动入库)
*/
@BatchOwner(Watcher.GaoJuan)
class OfficeFieldAddBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def sources = ["安居客app","办办网","房天下","中华写字楼", "点点租"]
//type:租售类别 wall:外墙
// def fieldsMap = ["elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","block":"block",
// "estate_type2":"property_type","estate_company":"property_company","estate_charge":"property_fee","all_area":"area",
// "standard_area":"standard_area","sight":"open_area","build_area":"build_area","build_type":"build_type","total_floor":"total_floor",
// "finish_building_date":"complete_time","loop_line_info":"ring_location","standard_height":"floor_height",
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","developer":"developer","fitment":"decoration","parking":"parking_cnt",
// "feature":"characteristic","description":"introduction","parking_fee":"car_fee","property_life":"property_life","bed_kind":"level",
// "lobby_height":"lobby_height","clear_height":"clear_height","foreign":"foreign","build_structure":"structure","address":"address",
// "lng":"location","open_date":"open_date","live_date":"live_date","real_use_percent":"room_rate","land_car_site":"land_car_site",
// "underground_car_site":"underground_car_site","land_floor":"land_floor","underground_floor":"underground_floor"]
def fieldsMap = ["build_area":"build_area"]
// "virescence_percent":"green_rate","cubage_percent":"volume_rate","estate_charge":"property_fee","finish_building_date":"complete_time",
// "bed_kind":"level","land_car_site":"land_car_site","underground_car_site":"underground_car_site","parking_fee":"car_fee",
// "elevators_count":"elevator_num","person_elevators_count":"elevator_num","goods_elevator_count":"elevator_num","clear_height":"clear_height",
// "foreign":"foreign","standard_area":"standard_area","foreign":"foreign","parking_fee":"car_fee","estate_company":"property_company","developer":"developer","all_area":"area",
static main(args) {
// runBatch(new OfficeFieldAddBatch(), args)
def task = new OfficeFieldAddBatch()
// task.fieldTempProcess()
task.deleteNotNormalOffice()
}
@Override
void process(Object args) {
def citys = ["深圳"]//CommonDatautils.findAllCity()
def fields = fieldsMap.keySet() as List
def analysRets = []
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
sources.each {source->
def esRets = findAllEsDataByCity(c, source)
def updateRets = processFieldsFromMatch(c , esRets, source)
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def analyRet = []
analyRet << source
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analysRets << analyRet
}
}
println analysRets.toString()
}
def processFieldsFromMatch(c,matchRets, source){
def fields = fieldsMap.keySet() as List
fields << "city"
def fieldValueMap = [:]
for(def esOffice : matchRets){
// if(esOffice.name!="东方建富时尚广场"){
// continue
// }
def office = CommonDatautils.findNormalDistrict(esOffice.city, esOffice.region, esOffice.name, "office", fields)
if(!office){
// println c+" "+esOffice.name+" 不存在"
continue
}
def newValues = strandField(c, office, esOffice, source)
if(newValues){
println office._id.toString()+" "+newValues.toString()
fieldValueMap.put(office._id.toString(), newValues)
}
}
return fieldValueMap;
}
// ["loop_line_info":"ring_location","feature":"characteristic","description":"introduction","lng":"location","open_date":"open_date"]
def strandField( c, office, esOffice, source){
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = office.get(seaweedfield)
if(!val){
def esVal = esOffice.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_type2"){
newVal = FiledStrand.processEstateType(esVal)
}
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), office.estate_type2)
if(newVal) {
newVal = newVal.toString()
}
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield == "loop_line_info"){
newVal = FiledStrand.processLoopLine(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area" || seaweedfield=="sight"){
newVal = FiledStrand.processArea(esVal.toString())
if(newVal){
newVal = newVal.toString()
}
}
if(seaweedfield =="car_site_count" || seaweedfield =="land_car_site" || seaweedfield =="underground_car_site"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, office.city)
}
if(seaweedfield =="block"){
newVal = FiledStrand.processBlock(office.city, office.region, esVal)
}
if(seaweedfield == "fitment"){
newVal = FiledStrand.processFitment(esVal)
}
if(seaweedfield == "build_type"){
newVal = FiledStrand.processBuildType(esVal, "office")
}
if(seaweedfield == "build_structure"){
newVal = FiledStrand.processBuildStructure(esVal)
}
if(seaweedfield == "bed_kind"){
newVal = FiledStrand.processLevel(esVal)
}
if(seaweedfield == "total_floor" || seaweedfield == "land_floor"){
newVal = FiledStrand.processFloor(esVal)
}
if(seaweedfield == "underground_floor"){
newVal = FiledStrand.processUndergroundFloor(esVal)
}
if(seaweedfield == "real_use_percent"){
newVal = FiledStrand.processRealUsePercent(esVal)
}
if(seaweedfield == "parking_fee"){
newVal = FiledStrand.processCarSiteRent(esVal)
println "[ ${source} ] old: "+esVal +" new:"+newVal
}
if(seaweedfield == "elevators_count"){
newVal = FiledStrand.processElevators(esVal)
}
if(seaweedfield == "person_elevators_count" && esVal.contains("客梯")){
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("客梯")))
}
if(seaweedfield == "goods_elevator_count" && esVal.contains("货梯")){
newVal = FiledStrand.processElevators(esVal.substring(esVal.indexOf("货梯")))
}
if(seaweedfield == "standard_height" || seaweedfield=="lobby_height" || seaweedfield=="clear_height"){
newVal = FiledStrand.processStandardHeight(esVal)
}
if(seaweedfield == "standard_area"){
newVal = FiledStrand.processStandardArea(esVal.toString())
}
if(seaweedfield == "finish_building_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "foreign"){
newVal = FiledStrand.processForeign(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
if(seaweedfield == "lng"){
try{
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
}
}catch(Exception e){
println esVal
}
}
}
}
}
return newValues;
}
def findAllEsDataByCity(city, source){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", source))
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "office2", queryBuilder);
}catch(Exception e){
datas = findAllEsDataByCity(city, source)
}
return datas;
}
def fieldUpdate(total,updateRets){
def countMap = [:]
for(def rm : updateRets){
def valMap = rm.value
fieldsMap.each{seaweedfield,esField->
if(valMap.containsKey(seaweedfield)) {
if(countMap.containsKey(seaweedfield)) {
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
}else{
countMap.put(seaweedfield, 1)
}
}
}
}
println countMap.toString()
def precentMap = [:]
fieldsMap.each() {key,skey->
if(countMap.containsKey(key)) {
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
}else {
precentMap.put(key, 0)
}
}
return precentMap;
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
def id = rm.key
def valMap = rm.value
BasicDBObject udata = new BasicDBObject()
valMap.each {k,v->
udata.put(k,v)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
}
}
def fieldTempProcess(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx",{d->
BasicDBObject udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("address", d['F'])
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
})
}
def deleteNotNormalOffice(){
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\需删除写字楼.xlsx",{d->
BasicDBObject udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("status", 1)
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])),new BasicDBObject('$set',udata))
})
}
}
@@ -74,11 +74,11 @@ class OfficeFieldCleanBatch extends BaseBatch{
// addBlock()
// addFieldValue()
// updateErrorField()
updateErrorField()
// export()
// test()
// addOffice()
fileAddBlock()
// fileAddBlock()
}
def export(){
@@ -134,33 +134,196 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
def updateErrorField(){
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistricts("ÉϺ£","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistrict("office", ["underground_car_site"])
// def offices = CommonDatautils.findAllDistricts("上海","office", ["build_area"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
for(def i=0;i<offices.size();i++){
final def o = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
if(o._id.toString()=="5954d66f9aed7b45d3785a40"){
println o._id.toString()
}
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
updateDistField(o)
println totalCount.get() +" "+o.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
}
def updateDistField(o){
def addr_alias = o.addr_alias
def udata = new BasicDBObject()
if(o.build_area){
def newVal = FiledStrand.processArea(o.build_area)
if(newVal && newVal.toString()!=o.build_area){
println o._id.toString()+" old:"+o.build_area +" new:"+newVal
udata.put("build_area", newVal.toString())
}
}
if(o.all_area){
def newVal = FiledStrand.processArea(o.all_area)
if(newVal.toString()!=o.all_area){
println o._id.toString()+" old:"+o.all_area +" new:"+newVal
udata.put("all_area", newVal.toString())
}
}
if(o.standard_area){
def newVal = FiledStrand.processStandardArea(o.standard_area)
if(newVal.toString()!=o.standard_area){
println o._id.toString()+" old:"+o.standard_area +" new:"+newVal
udata.put("standard_area", newVal.toString())
}
}
if(o.standard_height){
def newVal = FiledStrand.processStandardHeight(o.standard_height)
if(newVal.toString()!=o.standard_height){
println o._id.toString()+" old:"+o.standard_height +" new:"+newVal
udata.put("standard_height", newVal.toString())
}
}
if(o.clear_height){
def newVal = FiledStrand.processStandardHeight(o.clear_height)
if(newVal.toString()!=o.clear_height){
println o._id.toString()+" old:"+o.clear_height +" new:"+newVal
if(newVal){
udata.put("clear_height", newVal.toString())
}else{
udata.put("clear_height", "")
}
}
}
if(o.lobby_height){
def newVal = FiledStrand.processLobbyHeight(o.lobby_height)
if(newVal.toString()!=o.lobby_height){
println o._id.toString()+" old:"+o.lobby_height +" new:"+newVal
if(newVal){
udata.put("lobby_height", newVal.toString())
}else{
udata.put("lobby_height", "")
}
}
}
if(o.cubage_percent){
def newVal = FiledStrand.processCubagePercent(o.cubage_percent)
if(newVal && newVal.toString()!=o.cubage_percent){
println o._id.toString()+" old:"+o.cubage_percent +" new:"+newVal
udata.put("cubage_percent", newVal.toString())
}
}
if(o.virescence_percent){
def newVal = FiledStrand.processVirescencePercent(o.virescence_percent)
if(newVal.toString()!=o.virescence_percent){
println o._id.toString()+" old:"+o.virescence_percent +" new:"+newVal
udata.put("virescence_percent", newVal.toString())
}
}
if(o.elevators_count){
def newVal1 = FiledStrand.processElevators(o.elevators_count)
if(newVal1.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal1
udata.put("elevators_count", newVal1.toString())
}
def newVal2 = null
if(o.elevators_count.contains("客梯")){
newVal2 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("客梯")))
if(newVal2.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal2
udata.put("person_elevators_count", newVal2.toString())
}
}
def newVal3 = null
if(o.elevators_count.contains("货梯")){
newVal3 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("货梯")))
if(newVal3.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal3
udata.put("goods_elevator_count", newVal3.toString())
}
}
if(newVal2 && newVal3){
def total = newVal2.toInteger() + newVal3.toInteger()
udata.put("elevators_count", total.toString())
println o._id.toString()+" 总电梯:"+total
}
}
if(o.goods_elevator_count){
def newVal = FiledStrand.processElevators(o.goods_elevator_count)
if(newVal.toString()!=o.goods_elevator_count){
println o._id.toString()+" old:"+o.goods_elevator_count +" new:"+newVal
udata.put("goods_elevator_count", newVal.toString())
}
}
if(o.person_elevators_count){
def newVal = FiledStrand.processElevators(o.person_elevators_count)
if(newVal.toString()!=o.person_elevators_count){
println o._id.toString()+" old:"+o.person_elevators_count +" new:"+newVal
if(newVal){
udata.put("person_elevators_count", newVal.toString())
}else{
udata.put("person_elevators_count", "")
}
}
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
if(newVal.toString()!=o.land_car_site){
println o._id.toString()+" old:"+o.land_car_site +" new:"+newVal
if(newVal){
udata.put("land_car_site", newVal.toString())
}else{
udata.put("land_car_site", "")
}
}
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
if(newVal.toString()!=o.underground_car_site){
println o._id.toString()+" old:"+o.underground_car_site +" new:"+newVal
if(newVal){
udata.put("underground_car_site", newVal.toString())
}else{
udata.put("underground_car_site", "")
}
}
}
if(o.is_register){
def newVal = FiledStrand.processRegister(o.is_register)
if(newVal.toString()!=o.is_register){
println o._id.toString()+" old:"+o.is_register +" new:"+newVal
udata.put("is_register", newVal.toString())
}
}
if(o.property_life){
def newVal = FiledStrand.processPropertyLife(o.property_life)
udata.put("property_life", newVal)
@@ -173,7 +336,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
if(o.finish_building_date){
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
udata.put("finish_building_date", newVal.toString())
if(newVal && newVal.toString()!=o.finish_building_date){
println o._id.toString()+" old:"+o.finish_building_date +" new:"+newVal
udata.put("finish_building_date", newVal.toString())
}
}
if(o.real_use_percent){
@@ -181,21 +347,14 @@ class OfficeFieldCleanBatch extends BaseBatch{
udata.put("real_use_percent", newVal.toString())
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
udata.put("underground_car_site", newVal.toString())
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
udata.put("land_car_site", newVal.toString())
}
if(o.parking_fee){
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
udata.put("parking_fee", newVal)
}
def addr_alias = o.addr_alias
if(addr_alias){
def addrs = []
for(def arr : addr_alias){
@@ -214,13 +373,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
}
// if(o.estate_charge && !o.estate_charge.isNumber()){
//// def val = o.estate_charge.toString().toDouble()
//// if(val>60 || val < 0.35){
//// println val
// udata.put("estate_charge", null)
//// }
// }
if(udata){
udata.put("m_date", new Date())
@@ -9,7 +9,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def city = "³£ÖÝ"
@@ -0,0 +1,414 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 根据链家数据修改小区库中字段
* 使用到中间映射表
*/
@BatchOwner(Watcher.GaoJuan)
class UpdateDistrictField extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
def sources = ["链家", "我爱我家", "安居客"]
def statMap = [:]
static main(args) {
runBatch(new UpdateDistrictField(), args)
}
@Override
void process(Object args) {
def sourceMap = [:]
sources.each {source->
def valMap = findThirdMatchDist(source)
println source +" "+valMap.size()
if(valMap){
sourceMap.put(source, valMap)
}
}
def margeResults = megerDatas(sourceMap)
updateDB(margeResults)
println "修改完成 "+new Date().toString()
}
def megerDatas(sourceMap){
def set = new HashSet()
sources.each {source->
def valMap = sourceMap.get(source)
set.addAll(valMap.keySet())
}
def lastValues = []
set.each {id->
def dbObj = CommonDatautils.findDistrictById(id)
def datas = []
sources.each {source->
def valMap = sourceMap.get(source)
def sourceEsObj = findCommunities(valMap.get(id))
if(sourceEsObj){
if(source == "链家"){
sourceEsObj.source = "lianjia"
}else if(source == "我爱我家"){
sourceEsObj.source = "wiwj"
}else if(source == "安居客"){
sourceEsObj.source = "anjuke"
}
datas << sourceEsObj
}
}
if(dbObj && datas){
def newValMap = getNewField2(datas, dbObj)
if(newValMap){
newValMap.put("id", id)
lastValues << newValMap
}
}
}
statMap.each {source,fieldMap->
println source +" "+fieldMap.size()
fieldMap.each{f,v->
println f+" "+ v
}
println "----------------------------------------"
}
return lastValues
}
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
// def lianjisIds = lianjiaMap.keySet()
// def wiwjIds = wiwjMap.keySet()
// def anjukeIds = anjukeMap.keySet()
// def set = new HashSet()
// set.addAll(lianjisIds)
// set.addAll(wiwjIds)
// set.addAll(anjukeIds)
//
// def lastValues = []
// def lianjiaUpdateCount = 0
// def wiwjUpdateCount = 0
// def anjukeUpdateCount = 0
// def lianjiaCountMap = [:]
// def wiwjCountMap = [:]
// def anjukeCountMap = [:]
// set.each {id->
// BasicDBObject newValMap = new BasicDBObject()
// fields.each {field->
// def lianjiaValMap = lianjiaMap.get(id)
// def wiwjValMap = wiwjMap.get(id)
// def anjukeValMap = anjukeMap.get(id)
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
// newValMap.put(field, lianjiaValMap.get(field))
// lianjiaUpdateCount++
// def count = lianjiaCountMap.get(field)
// if(!count){
// lianjiaCountMap.put(field, 1)
// }else{
// lianjiaCountMap.put(field, count + 1)
// }
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
// newValMap.put(field, wiwjValMap.get(field))
// wiwjUpdateCount++
// def count = wiwjCountMap.get(field)
// if(!count){
// wiwjCountMap.put(field, 1)
// }else{
// wiwjCountMap.put(field, count + 1)
// }
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
// newValMap.put(field, anjukeValMap.get(field))
// anjukeUpdateCount++
// def count = anjukeCountMap.get(field)
// if(!count){
// anjukeCountMap.put(field, 1)
// }else{
// anjukeCountMap.put(field, count + 1)
// }
// }
// }
// newValMap.put("id", id)
// lastValues << newValMap
// }
//
// lianjiaCountMap.each {k,v->
// println "[链家] "+k+" "+v
// }
//
// wiwjCountMap.each {k,v->
// println "[我爱我家] "+k+" "+v
// }
//
// anjukeCountMap.each {k,v->
// println "[安居客] "+k+" "+v
// }
// return lastValues
// }
def updateDB(margeResults){
for(def mr : margeResults){
def id = mr.get("id")
mr.remove("id")
mr.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
}
}
// def batchUpdateDist(source){
//// def rets = findThirdMatchDist(source)
// println "[${source}] 符保条件小区量:"+rets.size()
// def updateIndex = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(rets.size())
// def allUpdateList = new CopyOnWriteArrayList()
//
// def sourceMap = new ConcurrentHashMap()
//
// for(def i=0;i<rets.size();i++) {
// final def ret = rets.get(i)
//// fixedThreadPool.execute(
//// new Runnable() {
//// void run() {
// try {
// def newValMap = running(ret.url, ret.old_match_id)
// if(newValMap){
// sourceMap.put(ret.old_match_id, newValMap)
// allUpdateList << newValMap
// updateIndex.addAndGet(1)
// }
// }catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
//// }
//// }
//// )
// }
// latch.await()
// println "总修改量:"+updateIndex.get()
// def countMap = [:]
// for(def map : allUpdateList){
// map.each{k,v->
// def count = countMap.get(k)
// if(!count){
// countMap.put(k, 1)
// }else{
// countMap.put(k, count+1)
// }
// }
// }
//
// countMap.each {k,v->
// println k +" 更新量:"+v
// }
//
// println "---------------------------------------------------------"
// return sourceMap
// }
def running(url, old_match_id){
def esObj = findCommunities(url)
def dbObj = CommonDatautils.findDistrictById(old_match_id)
if(dbObj){
def newValMap = getNewField(esObj, dbObj)
return newValMap
}
return null
}
def getNewField(esObj, dbObj){
BasicDBObject newValMap = new BasicDBObject()
esObj.each{field, esVal->
def dbVal = dbObj.get(field)
def newVal = null
if(esVal && field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(esVal && field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(esVal && field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
if(newVal){
newValMap.put(field, newVal)
}
}
return newValMap
}
def getNewField2(esDatas, dbObj){
BasicDBObject newValMap = new BasicDBObject()
def lastValueMap = [:]
def haveValueMap = [:]
fields.each{field->
for(def esData : esDatas){
def esVal = esData.get(field)
def dbVal = dbObj.get(field)
def newVal = null
if(haveValueMap.containsKey(field)){
continue
}else{
if(lastValueMap.containsKey(field)){
continue
}
if(esVal){
if(field=="estate_charge" || field=="car_site_count") {
esVal = FiledStrand.getNumber(esVal)
}
if(dbVal && esVal.toString()==dbVal.toString()){
haveValueMap.put(field, esVal)
}else{
if(field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
}
}
if(newVal){
lastValueMap.put(field, newVal)
//以下为统计各网站各字段的更新情况
def fieldStatMap = statMap.get(esData.source)
if(!fieldStatMap){
fieldStatMap = [:]
statMap.put(esData.source, fieldStatMap)
}
def count = fieldStatMap.get(field)
if(!count){
fieldStatMap.put(field, 1)
}else{
fieldStatMap.put(field, count + 1)
}
}
}
}
}
return lastValueMap
}
def findThirdMatchDist(source){
def query = new BasicDBObject("source",source).append("old_match_way", 1)
def keys = new BasicDBObject()
keys.put("url", 1)
keys.put("old_match_id", 1)
keys.put("old_match_city", 1)
keys.put("old_match_district", 1)
keys.put("old_match_name", 1)
def idsMap = [:]
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
for(def db : dbcursor){
if(!db.old_match_id && db.old_match_name){
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
if(dbo){
idsMap.put(dbo._id.toString(), db.url)
}
}else{
idsMap.put(db.old_match_id, db.url)
}
}
return idsMap
}
static def findCommunities(url){
if(!url){
return null
}
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
def rets = []
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
}catch(Exception e){
findCommunities(url)
}
def data = [:]
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
data.put("household_count", d.households)
data.put("loudong_count", d.loudong_cnt)
data.put("developer", d.developer)
data.put("estate_company", d.property_company)
data.put("estate_charge", d.property_fee)
data.put("car_site_count", d.parking_cnt)
}
}
return data;
}
}
@@ -14,7 +14,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
class EsMonthHouseStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -26,7 +26,8 @@ class EsMonthHouseStat {
// task.statHouseUpdateCount()
// task.statDistHouse()
task.statRent()
// task.statRent()
// task.statOfficeHouse()
}
def statRent(){
@@ -201,6 +202,51 @@ class EsMonthHouseStat {
},0)
}
def statOfficeHouse(){
def citys = CommonDatautils.findAllCity()
def allLists = []
allLists << ["城市", "近一年写字楼出售量", "写字楼量", "近一年出售房源量大于3的写字楼量", "大于3写字楼量占比"]
citys.each{c->
def count = findHouseCountByCat(c, "office_sell",20240101, 20250201)
def districts = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
def distCount = 0
if(count){
distCount = findHouseCountByCat(dist.city, dist.region, alias, 20240101, 20250201, "office_sell")
if(distCount && distCount>=3){
disthouseCount++
}
}
// println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
// println districts.size()
def precent = 0
if(districts){
precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
}
def row = [c , count, districts.size(), disthouseCount, precent+"%"]
println row
allLists << row
}
ExcelUtils.write("D:\\数据\\写字楼\\全国写字楼情况占比统计.xlsx", allLists, "写字楼情况")
}
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
def aliasStr = null;
@@ -219,6 +265,23 @@ class EsMonthHouseStat {
return total
}
def findHouseCountByCat(def city, def region, def alias, def startDate, def endDate, cat){
def aliasStr = null;
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') "
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
@@ -240,6 +303,17 @@ class EsMonthHouseStat {
}
def findHouseCountByCat(def city, def cat, def startDate, def endDate){
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}

Some files were not shown because too many files have changed in this diff Show More