Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-12-18 10:31:20 +08:00
12 changed files with 446 additions and 54 deletions
@@ -36,18 +36,22 @@
com.fangjia.coresite.filters.AccessLogFilter
</filter-class>
</filter>
<filter>
<filter-name>AntiSpiderFilter</filter-name>
<filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
<init-param>
<param-name>targetBeanName</param-name>
<param-value>antiSpiderFilter</param-value>
</init-param>
<init-param>
<param-name>targetFilterLifecycle</param-name>
<param-value>true</param-value>
</init-param>
</filter>
<!--爬虫拦截-->
<!-- <filter>-->
<!-- <filter-name>AntiSpiderFilter</filter-name>-->
<!-- <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>-->
<!-- <init-param>-->
<!-- <param-name>targetBeanName</param-name>-->
<!-- <param-value>antiSpiderFilter</param-value>-->
<!-- </init-param>-->
<!-- <init-param>-->
<!-- <param-name>targetFilterLifecycle</param-name>-->
<!-- <param-value>true</param-value>-->
<!-- </init-param>-->
<!-- </filter>-->
<filter>
<filter-name>AnalyticsFilter</filter-name>
<filter-class>
@@ -90,11 +94,11 @@
<dispatcher>REQUEST</dispatcher>
</filter-mapping>
<filter-mapping>
<filter-name>AntiSpiderFilter</filter-name>
<url-pattern>/*</url-pattern>
<dispatcher>REQUEST</dispatcher>
</filter-mapping>
<!-- <filter-mapping>-->
<!-- <filter-name>AntiSpiderFilter</filter-name>-->
<!-- <url-pattern>/*</url-pattern>-->
<!-- <dispatcher>REQUEST</dispatcher>-->
<!-- </filter-mapping>-->
<filter-mapping>
<filter-name>AnalyticsFilter</filter-name>
@@ -3,7 +3,7 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>${cityName}房产在线评估 - 房价网 fangjia.com</title>
<meta name="keywords" content="${cityName}房产评估,${cityName}房产估价,${cityName}房产数据,${cityName}房价">
<meta name="keywords" content="${cityName}房产评估,${cityName}房产估价,${cityName}房产数据,${cityName}房价,${cityName}房产网">
<meta name="description" content="房价网免费提供${cityName}房产在线评估服务,通过二手房挂牌价、房屋面积、楼层、装修等因素和科学的算法评估二手房房价,同时提供${cityName}二手房房价涨跌走势。">
<#include "../jsconstants.ftl" />
@@ -80,11 +80,11 @@
<li class="<#if (controllerName == 'evaluation' || controllerName == 'report' || actionName == 'trend') || (controllerName=='industry' && actionName=='consulting') || (controllerName=='saleLeads' && actionName=='dataAbility') || (controllerName=='solution' && actionName=='paimai')>current</#if>">
<a>房产大数据</a>
<a>在线估值</a>
<div class="dn" style="width:130px;padding-left: 10px;">
<ul>
<li><a href="${cityUrl}/zoushi">全国房价查询</a></li>
<li><a href="${cityUrl}/gujia">在线房产估值</a></li>
<li><a href="${cityUrl}/zoushi">房价查询</a></li>
<li><a href="${cityUrl}/gujia">房产估值</a></li>
<#-- <li><a href="${cityUrl}/data/power">数据能力</a></li>-->
<#-- <li><a href="${cityUrl}/solution/paimai">法拍房</a></li>-->
<#-- <li><a href="${cityUrl}/industry/consulting">咨询研究</a></li>-->
@@ -3,7 +3,8 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
<meta name="baidu-site-verification" content="codeva-tQ1XnRBEAd" />
<meta name="baidu-site-verification" content="codeva-ps90FaNiSm" />
<#-- <meta name="baidu-site-verification" content="codeva-tQ1XnRBEAd" />-->
<title>选择房价网-为金融机构解决地址标准化难题_治理率高达90% - 房价网 fangjia.com</title>
<meta name="keywords" content="房价网,上海估家网络,估家科技,房产评估,房产估价,房产大数据">
<meta name="description" content="房价网自有地址数据库丰富,涵盖全国367个城市地区90万个楼盘,8000万POI数据">
@@ -47,14 +48,7 @@
<#-- </div>-->
<#--</div>-->
<<<<<<< HEAD
<#--fixedNav01-->
=======
<#--<div class="advert mobile01">-->
<#-- <a id="close">关闭</a>-->
<#-- <div class="left-ad"></div>-->
<#--</div>-->
>>>>>>> c7c2bae298951f01848f2b654c4c5e7fdba454c0
<div class="bannerIndex mobile01">
<div class="imgBoxIndex">
@@ -4,7 +4,7 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>银行洞察客户、网点布局、业务营销_选房价网 - 房价网 fangjia.com</title>
<meta name="keywords" content="信用卡拓客,银行高净值客户,银行网点布局,银行网点选址,银行业务营销,社区银行">
<meta name="keywords" content="信用卡拓客,银行高净值客户,银行网点布局,银行网点选址,银行业务营销,在线估值,智能评估">
<meta name="description" content="房价网_国有大行的选择,客户地址标准化,洞察高净值客户,银行网点选址,业务营销解决方案">
<#include "../jsconstants.ftl" />
@@ -21,4 +21,5 @@ var Constants = {
<meta http-equiv="Content-Security-Policy" content="upgrade-insecure-requests">
</#if>
<#include "baidu_analysis.ftl" />
@@ -18,8 +18,8 @@
<#assign _currentCityDisplay>${cityName}</#assign>
</#if>
<#assign PAGE_TITLE>${affectValue}房价走势 - 房价网 fangjia.com</#assign>
<#assign PAGE_KEYWORDS>${affectValue}房价走势,${affectValue}房产数据,${affectValue}房价</#assign>
<#assign PAGE_TITLE>${affectValue}房价走势 - ${cityName}房产网 fangjia.com</#assign>
<#assign PAGE_KEYWORDS>${affectValue}房价走势,${affectValue}房产数据,${affectValue}房价,二手房均价查询,楼盘估值,在线评估</#assign>
<#assign headDesc>房价网提供${affectValue}及时、权威的房价走势图,免费查询最新季度房价、最近半年房价走势、去年同期房价走势、以及${affectValue}房价历史走势等信息。</#assign>
<#assign PAGE_DESCRIPTION>${headDesc}</#assign>
@@ -80,7 +80,7 @@
<div class="index_bg01 mobile01">
<div class="index-box01"><p>房价网自2006年起累计在线估值次数</p></div>
<div class="index-box01"><div class="evalCount_div"><p><label id="evalCount">274,200,000</label><i style="font-style: normal;">&nbsp;&nbsp;次</i></p></div><div class="icon_01 dn"></div></div>
<div class="index-box01"><div class="evalCount_div"><p><label id="evalCount"></label><i style="font-style: normal;">&nbsp;&nbsp;次</i></p></div><div class="icon_01 dn"></div></div>
<div class="index-box01"><p>房产数据覆盖全国367个城市地区、80万楼盘、8000万房号库、70万楼盘字段、5000万POI数据。</p></div>
</div>
@@ -389,10 +389,12 @@
timeout: 100000,
success: function(model) {
// $("#evalCount").text(toThousands(model.evalCount));
var startNum = document.getElementById("evalCount").innerHTML;
// var startNum = document.getElementById("evalCount").innerHTML;
var startNum = toThousands(model.evalCount - 10000)
if (startNum == ''){
$("#evalCount").text(toThousands(model.evalCount));
}else{
$("#evalCount").text(startNum);
numScroll(startNum,model.evalCount);
}
},
@@ -597,9 +597,9 @@ class CommonDatautils {
keys.put("region", 1);
keys.put("name", 1);
keys.put("_id", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
Iterator<DBObject> iters = cursor.iterator()
def result = [];
def result = new HashSet();
try{
while(iters.hasNext()) {
DBObject obj = iters.next()
@@ -0,0 +1,323 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.json.JsonSlurper
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//补充库中没有详细地址的小区
class AddDistrictAddress extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new AddDistrictAddress(), args)
}
@Override
public void process(Object args) {
// def citys = CommonDatautils.findAllCity()
// def index = 0
// def allrets = []
// for(def c : citys){
// index ++
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
//
// def rets = []
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
// for(def d : noRoadInfoDists){
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
// }
// allrets.addAll(rets)
// if(allrets.size()>50000){
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
// allrets.clear()
// }
// }
updateDistAddr()
}
def updateDistAddr(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("address", d['J'])
def addr_alias = dist.addr_alias
if(!dist.addr_alias){
addr_alias = []
}
if(d['K'] && !addr_alias.contains(d['K'])){
addr_alias << d['K']
}
if(addr_alias){
udata.put("addr_alias", addr_alias)
}
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
d['L'] = "已更新"
}
})
}
def noDetailRoadInfoDistricts(districts){
def rets = []
for(def d : districts){
if(d.address){
def flag = DataHandleUtils.isStandardRoad(d.address)
if(!flag){
rets << d
}
}else{
rets << d
}
}
return rets;
}
def baiduMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = baiduProcess(d, fullCityName)
if(retMap){
d.baidu_address = retMap.new_addr
d.baidu_old_addr = retMap.old_addr
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def baiduProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
for(def name : alias){
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
if(temp_address){
retMap.old_addr = temp_address
}
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
continue
}
}
return retMap
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def dbcursor = BAIDU_POI_COLL.find(q)
def address = null
for(def d : dbcursor){
address = d.addr
}
return address;
}
def gaodeMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = gaodeProcess(d, fullCityName)
if(retMap){
d.gaode_addr = retMap.new_addr
d.gaode_old_addr = retMap.old_addr
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def gaodeProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
def address = null
for(def name : alias){
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
if(temp_addr){
retMap.old_addr = temp_addr
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
continue
}
}
}
return retMap
}
def findGaoDeDatas2(city, region, name){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
def body = """
{
"_source": {
"includes": ["cat", "address", "id","name"]
},
"query": {
"bool": {
"must": [
{
"term": {
"city" : "${city}"
}
},
{
"term": {
"district" : "${region}"
}
},
{
"match": {
"name": {"query": "${name}","operator": "and"}
}
}
// ,{
// "match": {
// "small_type": "住宅"
// }
// }
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def address = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
def d = hits[0]
if(d._source.cat.contains("住宅")){
def source = d._source
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
if(newName == name){
address = source.address
}
}
}
}
return address
}
}
@@ -40,7 +40,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
@Override
public void process(Object args) {
// //将新增数据同步过来
// addNemCommunity()
addNemCommunity()
//// //接口进行匹配
match()
@@ -52,9 +52,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
def index = 0
for(def c : citys){
index ++
if(index < 206){
continue
}
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
matchByCity(c, noMatchDatas)
@@ -123,9 +120,6 @@ class ThirdPriceMappingBatch extends BaseBatch{
def index = 0
for(def c : citys){
index ++
if(index < 70){
continue
}
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
def addCount = addDatasToDB(crawlDatas)
println c + " 新增:"+ addCount
@@ -0,0 +1,54 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class DistrictStatRentCleanBatch extends BaseBatch{
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
static main(args) {
runBatch(new DistrictStatRentCleanBatch(), args)
}
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def keys = CommonDatautils.getSimpleDistricts("district", c)
running(keys, c)
println c +" ÔËÐÐÍê³É"
}
}
def running(keys, c){
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
def fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("district_name", 1)
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
def delRets = []
for(def d : dbcursor){
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
println d.city+"@"+d.region+"@"+d.district_name
delRets << d._id
}
if(delRets.size()>=100){
def delquery = new BasicDBObject()
delquery.put("_id", new BasicDBObject('$in', delRets))
RENT_DIST_STAT_COLL.remove(delquery)
delRets.clear()
}
}
}
}
@@ -38,8 +38,8 @@ class AddDistImagesBatch extends BaseBatch{
@Override
public void process(Object args) {
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def citys = CommonUtils.findAllCity()
citys.each {city->
def communitys = findAllEsData2(city, mdate)
@@ -47,10 +47,9 @@ class AddDistImagesBatch extends BaseBatch{
//第三方匹配小区
matchCommunitys(communitys)
}
// statImgCount()
}
def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
@@ -106,8 +105,8 @@ class AddDistImagesBatch extends BaseBatch{
def total = communitys.size()
for(def i=0;i<communitys.size();i++){
final def d = communitys[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
@@ -136,11 +135,32 @@ class AddDistImagesBatch extends BaseBatch{
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
// latch.await()
return rets
}
def statImgCount(){
def q = new BasicDBObject()
def dbcursor = seaweed_fs_coll.find(q)
def out_pics_count = 0
def floor_count = 0
for(def d : dbcursor){
if(d.out_pics){
out_pics_count = out_pics_count + d.out_pics.size()
}
if(d.floorplan_pics){
d.floorplan_pics.each{k,v->
floor_count = floor_count + v.size()
}
}
}
println out_pics_count
println floor_count
}
}