Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
Jian Shen
2022-09-15 13:20:14 +08:00
39 changed files with 1217 additions and 192 deletions
@@ -257,4 +257,9 @@ public class GeoUtils {
ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0;
return ret;
}
public static void main(String[] args){
Map<String, Double> location = gps84ToGcj02(31.218024629574998,121.36562884455);
System.out.println(location.toString());
}
}
@@ -98,5 +98,13 @@ public class SolutionController {
public Object standardizationAction(FangjiaModel model) {
return model;
}
/**
* 法拍房解决方案
* @param model
* @return
*/
public Object paimaiAction(FangjiaModel model){return model;}
}
@@ -40,6 +40,8 @@
.footer{height:110px;background:#515151;}
.footer > p{font-size:12px;color:#D7D7D7;text-align:center;line-height:18px;padding-top:38px;}
.footer > p label{color:#D7D7D7;}
</style>
<script>
@@ -31,6 +31,7 @@
<li><a href="${wwwUrl}/industry/credit">征信机构</a></li>
<li><a href="${wwwUrl}/industry/corporation">行业企业</a></li>
<li><a href="${wwwUrl}/industry/consulting">咨询研究</a></li>
<li><a href="${wwwUrl}/solution/paimai">司法拍卖</a></li>
</ul>
<ul>
<li><a href="${wwwUrl}/solution/appraisal">贷前估值解决方案</a></li>
@@ -41,6 +42,7 @@
<li><a href="${wwwUrl}/solution/location">地址精准营销解决方案</a></li>
<li><a href="${wwwUrl}/solution/standardization">地址标准化解决方案</a></li>
<li><a href="${wwwUrl}/solution/api">API数据接口解决方案</a></li>
<li><a href="${wwwUrl}/solution/paimai">法拍房数据解决方案</a></li>
</div>
</li>
<li class="subnav05 <#if controllerName == 'product' || controllerName == 'cpdb' || (controllerName == 'solution' && actionName == 'apiDoc')>current</#if>">
@@ -133,6 +135,7 @@
<li><a href="${wwwUrl}/solution/location" class="<#if controllerName == 'solution' && actionName == 'location'>current</#if>">地址精准营销解决方案</a></li>
<li><a href="${wwwUrl}/solution/standardization" class="<#if controllerName == 'solution' && actionName == 'standardization'>current</#if>">地址标准化解决方案</a></li>
<li><a href="${wwwUrl}/solution/api" class="<#if controllerName == 'solution' && actionName == 'api'>current</#if>">API 数据接口解决方案</a></li>
<li><a href="${wwwUrl}/solution/paimai" class="<#if controllerName == 'solution' && actionName=='paimai'>current</#if>">法拍房数据解决方案</a></li>
</ul>
</li>
@@ -0,0 +1,189 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>地址精准营销解决方案 - 房价网 fangjia.com</title>
<meta name="keywords" content="司法拍卖,法拍数据,法拍房">
<meta name="description" content="房价网为各类大数据平台法拍大数据智能分析">
<#include "../jsconstants.ftl" />
<link href="${staticUrl}/style/fangjia/v5/common.css?v=${version}" rel="stylesheet">
<link href="${staticUrl}/style/fangjia/v5/main.css?v=${version}" rel="stylesheet">
<link href="${staticUrl}/style/fangjia/v5/dialogBox.css?v=${version}" rel="stylesheet">
<script src="${staticUrl}/js/fangjia/v5/jquery-1.7.2.min.js?v=${version}"></script>
<style>
.column > div p{width:550px;}
</style>
<script>
if((navigator.userAgent.match(/(iPad|iPhone|iPod|Android|ios)/i))){
document.write("<link href='${staticUrl}/style/fangjia/v5/mobile.css?v=${version}' rel='stylesheet'>");
}
</script>
<style>
@media(max-device-width:768px){
.column > div > div > p{padding:0;}
.banner {
height: 22.5rem;
}
body > .adBox p{width:92%;margin:0 auto;}
}
</style>
</head>
<body>
<#include "/header/v5/_header.ftl"/>
<div class="banner mobile01" style="background:url(${staticUrl}/images/fangjia/v5/post-loan_banner.jpg) no-repeat center center;">
<div>
<p class="p01"><small>法拍房数据解决方案</small></p>
<p class="p02"><label>专业、实时、多维度的法拍大数据智能分析</label></p>
<p>
<small>
数字解读法拍房市场变化趋势,预测押品处置价值,助力金融机构洞察法拍房市场的风控信号。<br/>
从银行押品管理的视角,对全国各地法拍房市场进行统计分析,为押品生命周期的各阶段提供新的风控维度。
</small>
</p>
<div class="btnGroup">
<a class="link_btn link01_btn large" data_id="1023" cat="solution">立即体验</a>
</div>
</div>
</div>
<div class="banner mobile02" style="background:url(${staticUrl}/images/fangjia/v5/post-loan_banner_m.jpg) no-repeat center center;background-size:100%;">
<div>
<p class="p01"><small>法拍房市场分析服务</small></p>
<p class="p02"><label>专业、实时、多维度的法拍大数据智能分析</label></p>
<p class="p02">
<small>
数字解读法拍房市场变化趋势,预测押品处置价值,助力金融机构洞察法拍房市场的风控信号。<br/>
从银行押品管理的视角,对全国各地法拍房市场进行统计分析,为押品生命周期的各阶段提供新的风控维度。
</small>
</p>
<div class="btnGroup" style="margin-top: -25px;">
<a class="link_btn link01_btn large" data_id="1023" cat="solution">立即体验</a>
</div>
</div>
</div>
<div class="columnBox firstOne firstOneM">
<p>服务介绍</p>
<div class="column">
<div class="column_left mobile01" style="margin-left:40px;">
<img src="${staticUrl}/images/fangjia/v5/paimai1.png" style="width: 200px;height: 195px;">
</div>
<div class="column_left mobile02">
<img src="${staticUrl}/images/fangjia/v5/paimai1.png">
</div>
<div class="mobile01" style="margin-top: 5px;margin-left: 80px;float: left">
<div>
<p><span>司法拍卖市场是不良资产处置的终端,其交易金额、数量价格等信息具有重要价值。随着互联网的普及,网络司法拍卖日益成为司法拍卖的主流形式。通过对各主要网络拍卖平台的公开数据进行统计分析,可以反映出各地的法拍房市场规模、标的价格、交易热度情况,预判各类房产在拍卖市场的流动性和成交折扣率变化趋势,作为银行抵押房产价值评估和贷后风控的重要依据。</span></p>
</div>
</div>
<div class="mobile02">
<div>
<p><span>司法拍卖市场是不良资产处置的终端,其交易金额、数量价格等信息具有重要价值。随着互联网的普及,网络司法拍卖日益成为司法拍卖的主流形式。通过对各主要网络拍卖平台的公开数据进行统计分析,可以反映出各地的法拍房市场规模、标的价格、交易热度情况,预判各类房产在拍卖市场的流动性和成交折扣率变化趋势,作为银行抵押房产价值评估和贷后风控的重要依据。</span></p>
</div>
</div>
</div>
</div>
<div class="column">
<p>法拍房数据分析案例</p>
</div>
<div class="columnBox firstOne firstOneM" style="padding: 30px 0 0 0;">
<div class="column">
<div class="column_center2 mobile01">
<p><label>XX市法拍房市场综述</label></p>
<p>2022年上半年,XX市法拍房市场共计挂拍上拍房源有1265套,成交法拍房482套,成交率为38%(除去撤拍套数)。平均成交折扣折73%,其中住宅成交折扣率88%,别墅折扣率为65%,商办折扣率为60%,共计释放15.28亿元购房差价。</p>
<p><label>XX区法拍房市场综述</label></p>
<p>XX区法拍房市场2022年1-6月XX区司法拍卖市场挂拍住宅用房房源92套,成交48套,成交率52.2%,成交均价22450元,住宅成交折扣7.8折;挂拍商业用房房源364套,成交113套,成交率31%,成交均价44168,成交折扣9.4折;工业用房和土地无挂拍。</p>
</div>
<div class="column_center mobile02">
<p><label>XX市法拍房市场综述</label></p>
<p>2022年上半年,XX市法拍房市场共计挂拍上拍房源有1265套,成交法拍房482套,成交率为38%(除去撤拍套数)。平均成交折扣折73%,其中住宅成交折扣率88%,别墅折扣率为65%,商办折扣率为60%,共计释放15.28亿元购房差价。</p>
<p><label>XX区法拍房市场综述</label></p>
<p>XX区法拍房市场2022年1-6月XX区司法拍卖市场挂拍住宅用房房源92套,成交48套,成交率52.2%,成交均价22450元,住宅成交折扣7.8折;挂拍商业用房房源364套,成交113套,成交率31%,成交均价44168,成交折扣9.4折;工业用房和土地无挂拍。</p>
</div>
</div>
</div>
<div class="columnBox firstOne firstOneM">
<div class="column">
<div class="column_center">
<img src="${staticUrl}/images/fangjia/v5/paimai_2.png" class="mobile01">
<img src="${staticUrl}/images/fangjia/v5/paimai_2.png" class="mobile02 mobile02a" style="margin-bottom:3rem;">
</div>
<div class="column_center">
<img src="${staticUrl}/images/fangjia/v5/paimai_3.png" class="mobile01">
<img src="${staticUrl}/images/fangjia/v5/paimai_3.png" class="mobile02 mobile02a" style="margin-bottom:3rem;">
</div>
<div class="column_center">
<img src="${staticUrl}/images/fangjia/v5/paimai_5.png" class="mobile01">
<img src="${staticUrl}/images/fangjia/v5/paimai_5.png" class="mobile02 mobile02a" style="margin-bottom:3rem;">
</div>
<div class="column_center">
<img src="${staticUrl}/images/fangjia/v5/paimai_6.png" class="mobile01">
<img src="${staticUrl}/images/fangjia/v5/paimai_6.png" class="mobile02 mobile02a" style="margin-bottom:3rem;">
</div>
<div class="column_center">
<img src="${staticUrl}/images/fangjia/v5/paimai_7.png" class="mobile01">
<img src="${staticUrl}/images/fangjia/v5/paimai_7.png" class="mobile02 mobile02a" style="margin-bottom:3rem;">
</div>
</div>
</div>
<div class="adBox adBox01 mobile01">
<p style="margin-top: -10px"><label>法拍房数据解决方案</label></p>
<p><small>如想获取您所关注城市的法拍房市场分析报告,或者查询任一楼盘的法拍成交折扣率,请致电&nbsp;&nbsp;<b style="font-size: 18px;color: yellow;">400-188-3787</b>&nbsp;&nbsp;联系我们,或者留下您的联系方式和需求,我们会第一时间联系您。</small></p>
<a class="link_btn link01_btn" data_id="1023" cat="solution" >立即体验</a>
</div>
<div class="adBox adBox01 mobile02">
<p><label>法拍房数据解决方案</label></p>
<p><small>如想获取您所关注城市的法拍房市场分析报告,或者查询任一楼盘的法拍成交折扣率,请致电<label>400-188-3787</label>联系我们,或者留下您的联系方式和需求,我们会第一时间联系您。</small></p>
<a class="link_btn link01_btn" data_id="1023" cat="solution">立即体验</a>
</div>
<div class="resource" data-url = "${wwwUrl}/resource/recommend?module=1015"></div>
<div class="adBox adBox02">
<div>
<div>
<p><label>CPDB房产大数据库</label></p>
<p><small>欢迎亲自体验CPDB房产大数据库,您可通过在线方式申请测试数据或联系数据顾问,他们将非常乐意为您的数据需求量身定制合适的解决方案。</small></p>
</div>
<div class="btnGroup">
<a class="link_btn link01_btn" href="${wwwUrl}/cpdb/">CPDB大数据</a>
<a class="link_btn link01_btn" href="${wwwUrl}/solution/api">API解决方案</a>
</div>
</div>
</div>
<#include "/header/v5/_footer.ftl"/>
<script src="${staticUrl}/js/fangjia/v5/main.js?v=${version}"></script>
</body>
</html>
@@ -27,8 +27,8 @@ public enum NewResourceRecommendEnum {
API_GUZ_HI("1019", "AMP估价管理平台"),
FJT("1020", "房价通"),
FANG_JIA_ZOU_SHI("1021", "房价走势"),
FDANG_DIAO_BAO_GAO("1022", "房调报告");
FDANG_DIAO_BAO_GAO("1022", "房调报告"),
FA_PAI_FANG("1023", "法拍房数据解决方案");
NewResourceRecommendEnum(String id, String value) {
@@ -23,7 +23,8 @@ public enum SalesLeadEnum {
FDANG_DIAO_BAO_GAO("1017", "房产专业调查评估报告"),
BUSINESS_ASK("1018", "业务咨询"),
HOUSE_EVAL("1019", "房产评估"),
DIDATA("1020", "地数平台");
DIDATA("1020", "地数平台"),
FA_PAI_FANG("1023", "法拍房数据解决方案");
@@ -649,7 +649,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
jsonList.add(temp);
}
if(StringUtils.isNotBlank(query.getSort())){
if(StringUtils.isNotBlank(query.getSortField())){
JSONObject jsonOrder = new JSONObject();
if(query.getOrder()){
jsonOrder.put("order", "asc");
@@ -77,8 +77,16 @@ class CcbCommonUtils {
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def rets = []
def sets = new HashSet()
for(def d : dbcursor){
rets << d
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
if(!sets.contains(key)){
rets << d
sets << key
}else{
println key
}
}
return rets
}
@@ -370,24 +370,25 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys
def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202206
def month = 202207
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202207)
def rets = findPeGteThirdParty(city, 202208)
def total = 0
println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
def cityId = cityCodeDatas.get(city)
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
@@ -0,0 +1,193 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class FjwPricePkCcbPrice {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new FjwPricePkCcbPrice()
// task.run()
task.updatePrice()
}
def updatePrice(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d->
index++
println index
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
if(dbo){
d['G'] = dbo.future_value
}
})
}
def run(){
def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size()
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
println "fjw总量:"+ccbPrices.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(ccbPrices.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<ccbPrices.size();i++){
final def rMap = ccbPrices[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def row = []
def dist = CommonDatautils.findDistrictById(rMap.commid)
if(dist){
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
def fjwPrice = null
def fjwCalPrice = null
if(priceMap){
fjwPrice = priceMap.avg_price
fjwCalPrice = priceMap.calc_value
}
row << rMap.commid
row << dist.city
row << dist.region
row << dist.name
row << fjwPrice
row << fjwCalPrice
if(priceMap && priceMap.value_type){
if(priceMap.value_type == 1){
row << "指导价"
}else if(priceMap.value_type == 2){
row << "自身样本"
}else if(priceMap.value_type == 3){
row << "第三方"
}else if(priceMap.value_type == 4){
row << "交易数据"
}else if(priceMap.value_type == 5){
row << "周边楼盘"
}else if(priceMap.value_type == 6){
row << "推导"
}else if(priceMap.value_type == 9){
row << "神经网络"
}else if(priceMap.value_type == 10){
row << "无方式"
}else{
row << ""
}
}else{
row << ""
}
row << rMap.avg_price
row << rMap.calc_type
if(fjwPrice && rMap.avg_price){
def precent = (fjwPrice - rMap.avg_price)*100/rMap.avg_price
row << DataHandleUtils.getDoubleRound(2, precent)
}else{
row << ""
}
rets << row
}else{
println rMap.commId + " 没找到小区"
}
println totalCount.get()+" "+ rMap.cityname +" "+rMap.commname
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await();
def headers = ["小区ID","城市","区域","小区名","房价网价格","房价网预期价格","房价网计算方式","建行价格","建行计算方式","价格差比(房价网价格-建行价格)*100/建行价格"]
batchCreateFile(rets, "D:\\数据\\建行\\测试\\房价网与建行均价比对.csv",headers,"GBK")
}
def getCommunityBasePriceGj(yearMonth) {
def sql = "select CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE from community_base_price_gj where year_month=${yearMonth} order by cityid";
def allRows = []
sqlInstance236.eachRow(sql, { r ->
def map = [:]
map.cityname = r.CITYNAME
map.distname = r.DISTNAME
map.commid = r.COMMID
map.commname = r.COMMNAME
map.year_month = r.YEAR_MONTH
map.calc_avg_price = r.CALC_AVG_PRICE
map.avg_price = r.AVG_PRICE
// 计算类型,1:人工估值 2:样本计算 3:神经网络
if(r.CALC_TYPE){
if(r.CALC_TYPE == 1){
map.calc_type = "人工估值"
}else if(r.CALC_TYPE == 2){
map.calc_type = "样本计算"
}else if(r.CALC_TYPE == 3){
map.calc_type = "神经网络"
}
}
allRows << map
});
return allRows
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "@|@"))
sb.append(System.getProperty("line.separator"));
println index
if(index%10000==0){
println index+"开始写入CSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"结束写入CSV "+new Date().toLocaleString()
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-21")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -30,8 +30,8 @@ import java.util.regex.*;
//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -197,6 +197,79 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//println "共查到:"+index
def index = 0
//def commIdThirdMap = findAllThird(202207, sqlInstance236);
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//cityCodeDatas.each{city,cityId->
//
// deleteThirdByCityId(cityId, 202207, sqlInstance236)
// println city + "完成"
//}
def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat"))
for(line in lines) {
def arrs = line.toString().split("\\|@\\|")
if(arrs){
def sql = "delete from guided_price_gj where id='${arrs[0]}'"
sqlInstance236.executeUpdate(sql)
}
}
def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){
def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}"
sqlInstance236.executeUpdate(sql)
}
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d ->
// if (d['C']) {
// index ++T
// def cityId = cityCodeDatas.get(d['A'])
// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236)
// if(sources.size()>1){
// if(sources.contains("安居客") && sources.contains("58同城")){
// sources.remove("58同城")
// }else if(sources.contains("链家") && sources.contains("贝壳")){
// sources.remove("贝壳")
// }
// }
// if(sources.size()==1){
// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId
// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236)
// }
// }
//},0)
def findAllThird(price_date, def sqlInstance236){
def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}"
def map = [:]
sqlInstance236.eachRow(sql,{r->
def sources = map.get(r.commid)
if(!sources){
sources = []
map.put(r.commid, sources)
}
sources << r.data_source
});
return map
}
def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){
def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}"
sqlInstance236.executeUpdate(sql)
}
def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){
def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}"
def sources = []
sqlInstance236.eachRow(sql,{r->
sources << r.data_source
});
return sources
}
def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){
def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}"
def count = 0
@@ -326,52 +399,52 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//
//},0)
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
// if(d['A']){
//// def region = d['B']
//// def newRegion = RegionData.formatRegion(d['A'], region)
//// if(newRegion) {
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("status", 1)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d
// }
//// }
// }
//},0)
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
// if(d['A']){
//// def region = d['B']
//// def newRegion = RegionData.formatRegion(d['A'], region)
//// if(newRegion) {
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("status", 1)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d
// }
}
},0)
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
// }
}
},0)
//// }
// }
//},0)
@@ -2049,6 +2049,20 @@ class CommonDatautils {
}
public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryDistrictPrices(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
@@ -0,0 +1,159 @@
package com.fangjia.script.crawl
import com.alibaba.fastjson.JSONArray
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
class SchoolLocation {
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
static main(args) {
def task = new SchoolLocation()
task.run()
}
def run(){
def query = new BasicDBObject()
query.put("province","江苏")
//query.put("lat",null)
query.put("lat",new BasicDBObject('$exists',false))
def cursor = school_parse_coll.find(query)
def index = 0
for(def d : cursor){
index++
def datas = getSchoolLatAndLng(d.city, d.name, d.address, false)
if(datas){
for(def ret : datas){
def addr = d.address
if(d.address.contains(d.city+"市")){
def cityStr = d.city+"市"
addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size())
}
if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){
def udata = new BasicDBObject()
udata.put("lat", ret.lat.toString().toDouble())
udata.put("lng", ret.lng.toString().toDouble())
udata.put("status", 0)
school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
println index +" "+udata
break;
}
}
}
}
}
def getSchoolLatAndLng(city, name, address, isFullMatch){
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","").replace("?","").replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null;
com.alibaba.fastjson.JSONObject result = null;
try {
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
String response = request.get(url, params, "UTF-8");
com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response)
if (jsonResponse != null) {
JSONArray ja = jsonResponse.getJSONArray("pois");
if(ja != null && ja.size() > 0) {
poiList = new ArrayList<>();
for(int i=0;i<ja.size();i++){
com.alibaba.fastjson.JSONObject jo_item = ja.getJSONObject(i);
if(jo_item.containsKey("cityname") && !jo_item.getString("cityname").contains(city)){
continue;
}
String item_address = jo_item.containsKey("address") ? jo_item.getString("address") : null;
String item_name = jo_item.containsKey("name") ? jo_item.getString("name") : null;
boolean getedData = false;
if(isFullMatch == true){
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.equalsIgnoreCase(item_name, address)){
getedData = true;
}
}else{
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.contains(item_address, address)
|| StringUtils.equalsIgnoreCase(item_name, address) || StringUtils.contains(item_name, address) || StringUtils.contains(address, item_name)){
getedData = true;
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(item_name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(item_address, address);
if(score1 || score2){
getedData = true;
}
if(!getedData){
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(address, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(item_name, " ")
if(py1 == py2){
getedData = true;
}
}
}
}
def type = jo_item.getString("type");
if(getedData == true){
com.alibaba.fastjson.JSONObject jo = new com.alibaba.fastjson.JSONObject();
if(jo_item.containsKey("id")){
jo.put("id", jo_item.getString("id"))
}
if(jo_item.containsKey("name")){
jo.put("name", jo_item.getString("name"))
}
if(jo_item.containsKey("type")){
jo.put("type", type)
}
if(jo_item.containsKey("address")){
jo.put("address", jo_item.getString("address").replace("[]",""))
}
if(jo_item.containsKey("pname")){
jo.put("pname", jo_item.getString("pname"))
}
if(jo_item.containsKey("cityname")){
jo.put("cityname", jo_item.getString("cityname"))
}
if(jo_item.containsKey("adname")){
jo.put("adname", jo_item.getString("adname"))
}
if(jo_item.containsKey("location")){
def arrs = jo_item.getString("location").split(",")
jo.put("lat", arrs[1])
jo.put("lng", arrs[0])
}
if(type && type.contains("出入口")){
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】-${type}"
}else{
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】"
poiList.add(jo);
}
}else{
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
}
}
}else{
println "X 高德地图 ( "+city+"-"+address+" )无结果!"
}
}
} catch (Exception e) {
e.printStackTrace();
}
return poiList;
}
}
@@ -136,7 +136,7 @@ class FzDataResultStat {
def regionPriceTrendRun(){
def city = "福州"
def add = -6;
def add = -9;
def months = []
while(add<0){
months << DateUtils.addMonth(new Date(), add)
@@ -149,7 +149,7 @@ class FzDataResultStat {
def exportRegion(city, months){
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","闽侯","福清","连江","闽清","罗源","永泰","平潭"];
def datas =[]
def header = ["城市", "区域"]
for(def m : months){
@@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{
// int endDate = 20191201
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd")
// def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
def endDate = "20220710"
def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
// QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false);
@@ -110,6 +110,7 @@ class FangjiaClientUtils {
public static def saveToEsPaimai(def data){
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr)
}
@@ -376,13 +376,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("福州")){
if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
// }
// cityMap.put(e.text(), "https:"+href)
}
}
}
return list
@@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.NumberUtils
import java.util.Map.Entry;
import java.util.regex.Pattern
@@ -175,22 +177,23 @@ class ExportPaimai {
def run(){
def city = "苏州"
def region = "姑苏"
def city = "南京"
def region = null
def datas = findPaimaiData(city,region)
println "总量:"+datas.size()
def rows = []
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型",
"映射资产类型","映射拍卖状态","URL"]
rows << header
// def set = new HashSet()
def set = new HashSet()
def index = 0
for(def d : datas){
// if(!set.contains(d.title)){
if(!set.contains(d.title)){
index++
println index
// set << d.title
set << d.title
def row = []
row << d.title.replace(",",",")
row << d.province
@@ -226,14 +229,19 @@ class ExportPaimai {
row << d.lookCount
row << d.callCount
row << d.delayCount
row << d.url
row << d.rateDiscount//折价率
row << d.mappingEstateType;//映射类型
row << d.zcType;//拍卖类型:资产拍卖 司法拍卖
row << d.mappingAgencyName//映射资产类型
row << d.mappingStatus//映射拍卖状态
row << d.url
rows << row
// }else{
// println d.title
// }
}else{
println d.title +" "+d.url
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}-姑苏-撤回中止).xlsx",rows,"数据")
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\南京法拍.xlsx",rows,"数据")
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
println "生成完成........"
@@ -452,12 +460,12 @@ class ExportPaimai {
if(region){
pmquery.put("region",region)
}
pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pmquery.put("url", new BasicDBObject('$regex', pattern))
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
@@ -465,6 +473,28 @@ class ExportPaimai {
for(def db : dbcursor){
rets << db
}
// Collections.sort(rets, new Comparator<BasicDBObject>() {
// @Override
// int compare(BasicDBObject o1, BasicDBObject o2) {
// if(o1.finishDate && o2.finishDate){
// return -o1.finishDate.compareTo(o2.finishDate)
// }else{
// return 0
// }
//
// }
// })
rets.sort {a,b->
if(a.finishDate && b.finishDate){
def v1 = a.finishDate
def v2 = b.finishDate
return v1.compareTo(v2)
}else{
return 0
}
}
rets = rets.reverse()
return rets
}
}
@@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome {
def standardRegion(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
@@ -152,8 +152,9 @@ public class LianjiaCrawlByChrome {
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("city", "福州")
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
@@ -187,8 +188,19 @@ public class LianjiaCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
driver.get(cityUrl+"chengjiao/")
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -224,12 +236,12 @@ public class LianjiaCrawlByChrome {
}
def getPageDeail(driver, city, cityUrl){
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
def url = cityUrl + "pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
@@ -241,7 +253,7 @@ public class LianjiaCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
parse(city, e, regionText)
}else{
stop = true;
break;
@@ -259,7 +271,7 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e){
def parse(city,e, regionText){
try {
def title = e.select(".title a").text()
if (title) {
@@ -267,7 +279,11 @@ public class LianjiaCrawlByChrome {
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
@@ -275,50 +291,61 @@ public class LianjiaCrawlByChrome {
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -446,9 +473,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
// }
}
}
}
return cityMap
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
@@ -33,6 +34,7 @@ class PaimaiDataSend {
static main(args) {
def task = new PaimaiDataSend()
// task.jdRunByMongo()
// task.jdRun()
// task.aliZcRun()
task.aliZcRun2()
@@ -64,7 +66,8 @@ class PaimaiDataSend {
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -95,11 +98,22 @@ class PaimaiDataSend {
}
}
def jdRun(){
def jdRunByMongo(){
def pm_q = new BasicDBObject()
pm_q.put("city","宿迁")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
pm_q.put("mappingStage", null)
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
sendJd(d.url)
}
}
def sendJd(url){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁市")
// q.put("url","https://paimai.jd.com/108245329")
q.put("url", url)
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -110,6 +124,45 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("content_json", d.content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content.content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
def jdRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
// q.put("parser_status", new BasicDBObject('$ne',3))
q.put("url","https://paimai.jd.com/115669796")
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
println "jd:"+index
JSON json = new JSONObject()
json.put("project", "paimai_jd")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
@@ -134,17 +187,19 @@ class PaimaiDataSend {
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city","苏州")
pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// pm_q.put("city","苏州")
// pm_q.put("region","姑苏")
pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("status", null)
// pm_q.put("mappingEstateType", null)
// pm_q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
def flag = sendAliZc(d.url)
// def flag = sendAliZc(d.url)
def flag = sendAli(d.url)
if(!flag){
println "不存在:"+d.url
}
@@ -194,15 +249,59 @@ class PaimaiDataSend {
return flag
}
def sendAli(def url){
def index = 0
def q = new BasicDBObject()
q.put("url",url)
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def flag = false
for(def d : dbcursor){
index++
flag = true
println "ali-fp: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
return flag
}
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
// q.put("city","宿迁")
q.put("url","https://sf-item.taobao.com/sf_item/665877683620.htm")
q.put("city",new BasicDBObject('$in', ["南京","宿迁"]))
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
// q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def rets = []
@@ -229,10 +328,20 @@ class PaimaiDataSend {
json.put("result", result)
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
println "共:"+rets.size()
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
}
def send(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
@@ -245,7 +354,7 @@ class PaimaiDataSend {
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("parser_status", 2)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
@@ -264,13 +373,18 @@ class PaimaiDataSend {
def aliZcRun2(){
def index = 0
def q = new BasicDBObject()
q.put("city","苏州")
q.put("city",new BasicDBObject('$in', ["上海"]))
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","姑苏")
// q.put("url","https://zc-item.taobao.com/auction/636884262340.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
// q.put("url","https://sf-item.taobao.com/sf_item/586415617368.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
index++
if(!d.url){
println d._id.toString()
continue
}
println "ali-zc: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao_zc")
@@ -292,35 +406,47 @@ class PaimaiDataSend {
result.put("region", d.region)
json.put("result", result)
rets << result
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
println "共:"+rets.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)//result2parse2dev
def udata = new BasicDBObject()
udata.put("parser_status", 4)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", json.get("_id")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
latch.await()
// println "共:"+rets.size()
//
// def totalCount = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(rets.size())
// for(def i=0;i<rets.size();i++){
// final def json = rets[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try{
// totalCount.addAndGet(1)
// byte[] packedMsg = MessageUtils.pack(json)
// CommonUtils.sendMessage("result2parse2dev", packedMsg)
//
//
// def udata = new BasicDBObject()
// udata.put("parser_status", 1)
// udata.put("m_date", new Date())
// ali_zc_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
// println "send: "+totalCount.get()
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// });
// }
// latch.await()
}
def paimaiSendToEs(){
@@ -349,8 +475,9 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
if(d.city){
@@ -367,7 +494,7 @@ class PaimaiDataSend {
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evaluatePrice
o.evaluatePrice = d.evalatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
@@ -396,6 +523,7 @@ class PaimaiDataSend {
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.tradeAvgPrice = d.tradeAvgPrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
@@ -406,6 +534,13 @@ class PaimaiDataSend {
}else if(d.url.contains("rmfysszc.gov.cn")){
o.source = "人民法院讼诉网"
}
o.rateDiscount = d.rateDiscount//折价率
o.mappingEstateType = d.mappingEstateType;//映射类型
o.paiMaiType = d.paiMaiType;//拍卖类型:资产拍卖 司法拍卖
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
o.mappingStatus = d.mappingStatus//映射拍卖状态
o.mappingStage = d.mappingStage
rets << o
}
}
@@ -28,11 +28,29 @@ class CityStat {
def task = new CityStat()
// task.run()
// task.run1();
task.run3();
// task.run3();
// task.run4()
// task.run6()
// task.run7()
// task.run8()
task.dealPriceStat()
}
def dealPriceStat(){
def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
def rets = []
rets << ["城市","2021-1月总量"]
def startDate = DateUtils.parseFromTightByPattern('2021-12-01','yyyy-MM-dd')
def endDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
for(def c : citys){
def ret = []
ret << c
ret << findDealPriceCount(c, startDate, endDate)
rets << ret
println ret
}
ExcelUtils.write("D:\\数据\\2021成交数据.xlsx", rets, "月统计")
}
// def run8(){
@@ -464,6 +482,14 @@ class CityStat {
return count;
}
def findDealPriceCount(def city, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
def count = DEAL_PRICE_COLL.count(q)
return count;
}
def calTradeDistCount(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
@@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
@@ -17,8 +18,19 @@ import org.bson.types.ObjectId
import java.text.SimpleDateFormat
def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
DBUtils.seaweed.save(dist)
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
def q = new BasicDBObject()
q.put("root", "anjuke.com")
def dbcursor = coll.find(q)
def map = [:]
for(def d : dbcursor){
map.put(d.city, d.city_short)
}
println map.size()
println map
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
//DBUtils.seaweed.save(dist)
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
@@ -0,0 +1,136 @@
package com.fangjia.script.gj.test
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018)
DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018)
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
//while (true){
// def q = new BasicDBObject()
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
// q.put("status",new BasicDBObject('$ne', 1))
// def dbcursor = school_coll.find(q).limit(1000)
// def rets = []
// for(def d : dbcursor){
// JSON json = new JSONObject()
// json.put("project", d.project)
// json.put("url", d.url)
//
// JSON result = new JSONObject()
// result.put("content", d.content)
// def jsonStr = d.map_info
//
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
// if(jsonStr.contains("\n")){
// jsonStr = jsonStr.replaceAll('\\n','')
// }
// result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
//
//
// result.put("type", d.type)
// json.put("result", result)
//
//
// rets << json
//// if(rets.size()==5000){
//// println "共:"+rets.size()
//// send(rets, fixedThreadPool, school_coll)
//// rets.clear()
//// }
// }
//
// if(rets.size()>0){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
// }
//}
def send(def rets, def fixedThreadPool, def school_coll){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def query = new BasicDBObject()
query.put("source","搜学网")
//query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
query.put("lat",null)
def cursor = school_parse_coll.find(query)
for(def o : cursor){
def q = new BasicDBObject()
// q.put("province","江苏省")
q.put("url",o.url)
def dbcursor = sxue_school_coll.find(q)
def index = 0
for(def d : dbcursor){
index++
println index
JSON json = new JSONObject()
json.put("project", d.project)
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
result.put("name", d.name)
result.put("list_item", d.list_item)
def jsonStr = d.map_info
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
if(jsonStr.contains("\n")){
jsonStr = jsonStr.replaceAll('\\n','')
}
result.put("json_data", jsonStr)
result.put("type", d.type)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
sxue_school_coll.update(new BasicDBObject("url", d.get("url")), new BasicDBObject('$set', udata))
}
}
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "365ÌÔ·¿"//
def citys = findCrawlCity(source, 202206);
def source = "我爱我家"//
def citys = findCrawlCity(source, 202207);
for(def city : citys){
def count1 = findCount(source, city, 202206);
def count2 = findCount(source, city, 202207);
def count1 = findCount(source, city, 202207);
def count2 = findCount(source, city, 202208);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -16,15 +16,12 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
//"fang.com":"搜房",
// def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
//,"城市房产":"城市房产"
//"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房",
def esSources = ["C21":"C21","365淘房":"365淘房"]//"吉屋":"吉屋","365淘房":"365淘房"
//"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"]
static main(args) {
def task = new SynchDistrictPrice()
// task.run()
@@ -32,13 +29,13 @@ class SynchDistrictPrice {
}
def run1(){
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
// citys.remove("上海")
// citys.remove("广州")
esSources.each{esSource, mgSource->
for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource)
def rets = findAllEsData(city, esSource, mgSource, mdate)
println city +" "+rets.size()
synchData(city, rets, mgSource)
}
@@ -46,10 +43,10 @@ class SynchDistrictPrice {
}
def findAllEsData(city,esSource, mgSource){
def findAllEsData(city,esSource, mgSource, mdate){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d->
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
def s = d.city +"#"+d.region+"#"+d.name
if(!names.contains(s)){
names << s
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202206);
def count2 = findTotalCount(source, 202207);
def count1 = findTotalCount(source, 202207);
def count2 = findTotalCount(source, 202208);
println source +" "+count1 +" "+count2
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 76 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 439 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 37 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

@@ -80,11 +80,15 @@
.adBox p small{line-height:26px;}
.adBox .btnGroup{overflow:hidden;width:230px;margin:0 auto;}
.adBox01{width:1200px;height:202px;background:url(../../../images/fangjia/v5/adBox01_bg.png) no-repeat;margin-bottom:0;}
.adBox01 p label{color:#fff;}
.adBox01 p small{color:#fff;}
.adBox01 .link01_btn{margin:0 auto;margin-top:20px;}
.adBox02{width:100%;height:90px;background:url(../../../images/fangjia/v5/adBox02_bg.png) no-repeat;margin-bottom:0;}
.adBox02 > div{width:1200px;height:100px;margin:0 auto;}
.adBox02 > div > div{float:left;}
@@ -167,6 +171,7 @@
.firstOne .column .column_center img{margin:0 auto;}
.firstOne .column .column_center > p{text-align:center;line-height:32px;font-size:18px;width:1000px;margin:40px auto;}
.firstOne .column .column_center2 > p{line-height:32px;font-size:18px;width:1000px;}
.firstOne .column .column_center > div{overflow:hidden;width:795px;margin:0 auto;}
.firstOne .column .column_center > div > img{float:left;margin:0 40px;}
.firstOne .column .column_center > div > div{float:left;}