Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -36,18 +36,22 @@
|
||||
com.fangjia.coresite.filters.AccessLogFilter
|
||||
</filter-class>
|
||||
</filter>
|
||||
<filter>
|
||||
<filter-name>AntiSpiderFilter</filter-name>
|
||||
<filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
|
||||
<init-param>
|
||||
<param-name>targetBeanName</param-name>
|
||||
<param-value>antiSpiderFilter</param-value>
|
||||
</init-param>
|
||||
<init-param>
|
||||
<param-name>targetFilterLifecycle</param-name>
|
||||
<param-value>true</param-value>
|
||||
</init-param>
|
||||
</filter>
|
||||
|
||||
<!--爬虫拦截-->
|
||||
<!-- <filter>-->
|
||||
<!-- <filter-name>AntiSpiderFilter</filter-name>-->
|
||||
<!-- <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>-->
|
||||
<!-- <init-param>-->
|
||||
<!-- <param-name>targetBeanName</param-name>-->
|
||||
<!-- <param-value>antiSpiderFilter</param-value>-->
|
||||
<!-- </init-param>-->
|
||||
<!-- <init-param>-->
|
||||
<!-- <param-name>targetFilterLifecycle</param-name>-->
|
||||
<!-- <param-value>true</param-value>-->
|
||||
<!-- </init-param>-->
|
||||
<!-- </filter>-->
|
||||
|
||||
|
||||
<filter>
|
||||
<filter-name>AnalyticsFilter</filter-name>
|
||||
<filter-class>
|
||||
@@ -90,11 +94,11 @@
|
||||
<dispatcher>REQUEST</dispatcher>
|
||||
</filter-mapping>
|
||||
|
||||
<filter-mapping>
|
||||
<filter-name>AntiSpiderFilter</filter-name>
|
||||
<url-pattern>/*</url-pattern>
|
||||
<dispatcher>REQUEST</dispatcher>
|
||||
</filter-mapping>
|
||||
<!-- <filter-mapping>-->
|
||||
<!-- <filter-name>AntiSpiderFilter</filter-name>-->
|
||||
<!-- <url-pattern>/*</url-pattern>-->
|
||||
<!-- <dispatcher>REQUEST</dispatcher>-->
|
||||
<!-- </filter-mapping>-->
|
||||
|
||||
<filter-mapping>
|
||||
<filter-name>AnalyticsFilter</filter-name>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<title>${cityName}房产在线评估 - 房价网 fangjia.com</title>
|
||||
<meta name="keywords" content="${cityName}房产评估,${cityName}房产估价,${cityName}房产数据,${cityName}房价">
|
||||
<meta name="keywords" content="${cityName}房产评估,${cityName}房产估价,${cityName}房产数据,${cityName}房价,${cityName}房产网">
|
||||
<meta name="description" content="房价网免费提供${cityName}房产在线评估服务,通过二手房挂牌价、房屋面积、楼层、装修等因素和科学的算法评估二手房房价,同时提供${cityName}二手房房价涨跌走势。">
|
||||
|
||||
<#include "../jsconstants.ftl" />
|
||||
|
||||
@@ -80,11 +80,11 @@
|
||||
|
||||
|
||||
<li class="<#if (controllerName == 'evaluation' || controllerName == 'report' || actionName == 'trend') || (controllerName=='industry' && actionName=='consulting') || (controllerName=='saleLeads' && actionName=='dataAbility') || (controllerName=='solution' && actionName=='paimai')>current</#if>">
|
||||
<a>房产大数据</a>
|
||||
<a>在线估值</a>
|
||||
<div class="dn" style="width:130px;padding-left: 10px;">
|
||||
<ul>
|
||||
<li><a href="${cityUrl}/zoushi">全国房价查询</a></li>
|
||||
<li><a href="${cityUrl}/gujia">在线房产估值</a></li>
|
||||
<li><a href="${cityUrl}/zoushi">房价查询</a></li>
|
||||
<li><a href="${cityUrl}/gujia">房产估值</a></li>
|
||||
<#-- <li><a href="${cityUrl}/data/power">数据能力</a></li>-->
|
||||
<#-- <li><a href="${cityUrl}/solution/paimai">法拍房</a></li>-->
|
||||
<#-- <li><a href="${cityUrl}/industry/consulting">咨询研究</a></li>-->
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
|
||||
<meta name="baidu-site-verification" content="codeva-tQ1XnRBEAd" />
|
||||
<meta name="baidu-site-verification" content="codeva-ps90FaNiSm" />
|
||||
<#-- <meta name="baidu-site-verification" content="codeva-tQ1XnRBEAd" />-->
|
||||
<title>选择房价网-为金融机构解决地址标准化难题_治理率高达90% - 房价网 fangjia.com</title>
|
||||
<meta name="keywords" content="房价网,上海估家网络,估家科技,房产评估,房产估价,房产大数据">
|
||||
<meta name="description" content="房价网自有地址数据库丰富,涵盖全国367个城市地区90万个楼盘,8000万POI数据">
|
||||
@@ -47,14 +48,7 @@
|
||||
<#-- </div>-->
|
||||
<#--</div>-->
|
||||
|
||||
<<<<<<< HEAD
|
||||
<#--fixedNav01-->
|
||||
=======
|
||||
<#--<div class="advert mobile01">-->
|
||||
<#-- <a id="close">关闭</a>-->
|
||||
<#-- <div class="left-ad"></div>-->
|
||||
<#--</div>-->
|
||||
>>>>>>> c7c2bae298951f01848f2b654c4c5e7fdba454c0
|
||||
|
||||
|
||||
<div class="bannerIndex mobile01">
|
||||
<div class="imgBoxIndex">
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<title>银行洞察客户、网点布局、业务营销_选房价网 - 房价网 fangjia.com</title>
|
||||
<meta name="keywords" content="信用卡拓客,银行高净值客户,银行网点布局,银行网点选址,银行业务营销,社区银行">
|
||||
<meta name="keywords" content="信用卡拓客,银行高净值客户,银行网点布局,银行网点选址,银行业务营销,在线估值,智能评估">
|
||||
<meta name="description" content="房价网_国有大行的选择,客户地址标准化,洞察高净值客户,银行网点选址,业务营销解决方案">
|
||||
<#include "../jsconstants.ftl" />
|
||||
|
||||
|
||||
@@ -21,4 +21,5 @@ var Constants = {
|
||||
<meta http-equiv="Content-Security-Policy" content="upgrade-insecure-requests">
|
||||
</#if>
|
||||
|
||||
|
||||
<#include "baidu_analysis.ftl" />
|
||||
|
||||
@@ -18,8 +18,8 @@
|
||||
<#assign _currentCityDisplay>${cityName}</#assign>
|
||||
</#if>
|
||||
|
||||
<#assign PAGE_TITLE>${affectValue}房价走势 - 房价网 fangjia.com</#assign>
|
||||
<#assign PAGE_KEYWORDS>${affectValue}房价走势,${affectValue}房产数据,${affectValue}房价</#assign>
|
||||
<#assign PAGE_TITLE>${affectValue}房价走势 - ${cityName}房产网 fangjia.com</#assign>
|
||||
<#assign PAGE_KEYWORDS>${affectValue}房价走势,${affectValue}房产数据,${affectValue}房价,二手房均价查询,楼盘估值,在线评估</#assign>
|
||||
<#assign headDesc>房价网提供${affectValue}及时、权威的房价走势图,免费查询最新季度房价、最近半年房价走势、去年同期房价走势、以及${affectValue}房价历史走势等信息。</#assign>
|
||||
<#assign PAGE_DESCRIPTION>${headDesc}</#assign>
|
||||
|
||||
@@ -80,7 +80,7 @@
|
||||
|
||||
<div class="index_bg01 mobile01">
|
||||
<div class="index-box01"><p>房价网自2006年起累计在线估值次数</p></div>
|
||||
<div class="index-box01"><div class="evalCount_div"><p><label id="evalCount">274,200,000</label><i style="font-style: normal;"> 次</i></p></div><div class="icon_01 dn"></div></div>
|
||||
<div class="index-box01"><div class="evalCount_div"><p><label id="evalCount"></label><i style="font-style: normal;"> 次</i></p></div><div class="icon_01 dn"></div></div>
|
||||
<div class="index-box01"><p>房产数据覆盖全国367个城市地区、80万楼盘、8000万房号库、70万楼盘字段、5000万POI数据。</p></div>
|
||||
</div>
|
||||
|
||||
@@ -389,10 +389,12 @@
|
||||
timeout: 100000,
|
||||
success: function(model) {
|
||||
// $("#evalCount").text(toThousands(model.evalCount));
|
||||
var startNum = document.getElementById("evalCount").innerHTML;
|
||||
// var startNum = document.getElementById("evalCount").innerHTML;
|
||||
var startNum = toThousands(model.evalCount - 10000)
|
||||
if (startNum == ''){
|
||||
$("#evalCount").text(toThousands(model.evalCount));
|
||||
}else{
|
||||
$("#evalCount").text(startNum);
|
||||
numScroll(startNum,model.evalCount);
|
||||
}
|
||||
},
|
||||
|
||||
+2
-2
@@ -597,9 +597,9 @@ class CommonDatautils {
|
||||
keys.put("region", 1);
|
||||
keys.put("name", 1);
|
||||
keys.put("_id", 1);
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
|
||||
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
|
||||
Iterator<DBObject> iters = cursor.iterator()
|
||||
def result = [];
|
||||
def result = new HashSet();
|
||||
try{
|
||||
while(iters.hasNext()) {
|
||||
DBObject obj = iters.next()
|
||||
|
||||
+323
@@ -0,0 +1,323 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.json.JsonSlurper
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
//补充库中没有详细地址的小区
|
||||
class AddDistrictAddress extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictAddress(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def index = 0
|
||||
// def allrets = []
|
||||
// for(def c : citys){
|
||||
// index ++
|
||||
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
|
||||
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
|
||||
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
|
||||
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
|
||||
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
|
||||
//
|
||||
// def rets = []
|
||||
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
|
||||
// for(def d : noRoadInfoDists){
|
||||
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
|
||||
// }
|
||||
// allrets.addAll(rets)
|
||||
// if(allrets.size()>50000){
|
||||
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
|
||||
// allrets.clear()
|
||||
// }
|
||||
// }
|
||||
|
||||
updateDistAddr()
|
||||
|
||||
}
|
||||
|
||||
def updateDistAddr(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", d['J'])
|
||||
def addr_alias = dist.addr_alias
|
||||
if(!dist.addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
if(d['K'] && !addr_alias.contains(d['K'])){
|
||||
addr_alias << d['K']
|
||||
}
|
||||
|
||||
if(addr_alias){
|
||||
udata.put("addr_alias", addr_alias)
|
||||
}
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
d['L'] = "已更新"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def noDetailRoadInfoDistricts(districts){
|
||||
def rets = []
|
||||
for(def d : districts){
|
||||
if(d.address){
|
||||
def flag = DataHandleUtils.isStandardRoad(d.address)
|
||||
if(!flag){
|
||||
rets << d
|
||||
}
|
||||
}else{
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = baiduProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.baidu_address = retMap.new_addr
|
||||
d.baidu_old_addr = retMap.old_addr
|
||||
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
def baiduProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
|
||||
for(def name : alias){
|
||||
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
|
||||
if(temp_address){
|
||||
retMap.old_addr = temp_address
|
||||
}
|
||||
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
|
||||
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
|
||||
def dbcursor = BAIDU_POI_COLL.find(q)
|
||||
def address = null
|
||||
for(def d : dbcursor){
|
||||
address = d.addr
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
|
||||
def gaodeMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = gaodeProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.gaode_addr = retMap.new_addr
|
||||
d.gaode_old_addr = retMap.old_addr
|
||||
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
|
||||
def gaodeProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
def address = null
|
||||
for(def name : alias){
|
||||
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
|
||||
if(temp_addr){
|
||||
retMap.old_addr = temp_addr
|
||||
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
|
||||
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def findGaoDeDatas2(city, region, name){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
|
||||
def body = """
|
||||
{
|
||||
"_source": {
|
||||
"includes": ["cat", "address", "id","name"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"city" : "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"district" : "${region}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"name": {"query": "${name}","operator": "and"}
|
||||
}
|
||||
}
|
||||
// ,{
|
||||
// "match": {
|
||||
// "small_type": "住宅"
|
||||
// }
|
||||
// }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
|
||||
def address = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
def d = hits[0]
|
||||
if(d._source.cat.contains("住宅")){
|
||||
def source = d._source
|
||||
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
|
||||
if(newName == name){
|
||||
address = source.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return address
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+1
-7
@@ -40,7 +40,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// //将新增数据同步过来
|
||||
// addNemCommunity()
|
||||
addNemCommunity()
|
||||
//// //接口进行匹配
|
||||
match()
|
||||
|
||||
@@ -52,9 +52,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 206){
|
||||
continue
|
||||
}
|
||||
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
|
||||
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||
matchByCity(c, noMatchDatas)
|
||||
@@ -123,9 +120,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 70){
|
||||
continue
|
||||
}
|
||||
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
|
||||
def addCount = addDatasToDB(crawlDatas)
|
||||
println c + " 新增:"+ addCount
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictStatRentCleanBatch extends BaseBatch{
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictStatRentCleanBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def keys = CommonDatautils.getSimpleDistricts("district", c)
|
||||
running(keys, c)
|
||||
println c +" ÔËÐÐÍê³É"
|
||||
}
|
||||
}
|
||||
|
||||
def running(keys, c){
|
||||
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
|
||||
def fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("district_name", 1)
|
||||
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
|
||||
def delRets = []
|
||||
for(def d : dbcursor){
|
||||
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
|
||||
println d.city+"@"+d.region+"@"+d.district_name
|
||||
delRets << d._id
|
||||
}
|
||||
if(delRets.size()>=100){
|
||||
def delquery = new BasicDBObject()
|
||||
delquery.put("_id", new BasicDBObject('$in', delRets))
|
||||
RENT_DIST_STAT_COLL.remove(delquery)
|
||||
delRets.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+30
-10
@@ -38,8 +38,8 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
def citys = CommonUtils.findAllCity()
|
||||
citys.each {city->
|
||||
def communitys = findAllEsData2(city, mdate)
|
||||
@@ -47,10 +47,9 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
//第三方匹配小区
|
||||
matchCommunitys(communitys)
|
||||
}
|
||||
|
||||
// statImgCount()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findAllEsData2(city, mdate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||
|
||||
@@ -106,8 +105,8 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
def total = communitys.size()
|
||||
for(def i=0;i<communitys.size();i++){
|
||||
final def d = communitys[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
||||
@@ -136,11 +135,32 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
// latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def statImgCount(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = seaweed_fs_coll.find(q)
|
||||
def out_pics_count = 0
|
||||
def floor_count = 0
|
||||
for(def d : dbcursor){
|
||||
if(d.out_pics){
|
||||
out_pics_count = out_pics_count + d.out_pics.size()
|
||||
}
|
||||
if(d.floorplan_pics){
|
||||
d.floorplan_pics.each{k,v->
|
||||
floor_count = floor_count + v.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
println out_pics_count
|
||||
println floor_count
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user