Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -72,18 +72,18 @@
|
|||||||
<#include "/header/v6/_header.ftl"/>
|
<#include "/header/v6/_header.ftl"/>
|
||||||
<#include "/evaluation/checkUserNewRedirct.ftl"/>
|
<#include "/evaluation/checkUserNewRedirct.ftl"/>
|
||||||
|
|
||||||
<div class="bannerIndex mobile01">
|
<#-- <div class="bannerIndex mobile01">-->
|
||||||
<div class="imgBoxIndex">
|
<#-- <div class="imgBoxIndex">-->
|
||||||
<div class="bannerIndex01">
|
<#-- <div class="bannerIndex01">-->
|
||||||
<div style="text-align: left;margin-left: 50%;margin-right: 10%">
|
<#-- <div style="text-align: left;margin-left: 50%;margin-right: 10%">-->
|
||||||
<p><label>全国房产在线估值</label></p>
|
<#-- <p><label>全国房产在线估值</label></p>-->
|
||||||
<div class="btnGroup" align="left" style="float: left">
|
<#-- <div class="btnGroup" align="left" style="float: left">-->
|
||||||
<center><a id="a1" class="link_btn1 link01_btn text20" data_id="1040" cat="solution">申请试用</a></center>
|
<#-- <center><a id="a1" class="link_btn1 link01_btn text20" data_id="1040" cat="solution">申请试用</a></center>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
|
|
||||||
|
|
||||||
<div class="bannerIndex mobile02">
|
<div class="bannerIndex mobile02">
|
||||||
|
|||||||
@@ -52,18 +52,18 @@
|
|||||||
|
|
||||||
<div id="title" class="dn">${affectValue}</div>
|
<div id="title" class="dn">${affectValue}</div>
|
||||||
|
|
||||||
<div class="bannerIndex mobile01">
|
<#--<div class="bannerIndex mobile01">-->
|
||||||
<div class="imgBoxIndex">
|
<#-- <div class="imgBoxIndex">-->
|
||||||
<div class="bannerIndex01">
|
<#-- <div class="bannerIndex01">-->
|
||||||
<div style="text-align: left;margin-left: 50%;margin-right: 10%">
|
<#-- <div style="text-align: left;margin-left: 50%;margin-right: 10%">-->
|
||||||
<p><label>全国房价在线查询</label></p>
|
<#-- <p><label>全国房价在线查询</label></p>-->
|
||||||
<div class="btnGroup" align="left" style="float: left">
|
<#-- <div class="btnGroup" align="left" style="float: left">-->
|
||||||
<center><a id="a1" class="link_btn1 link01_btn text20" data_id="1039" cat="solution">申请试用</a></center>
|
<#-- <center><a id="a1" class="link_btn1 link01_btn text20" data_id="1039" cat="solution">申请试用</a></center>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#-- </div>-->
|
||||||
</div>
|
<#--</div>-->
|
||||||
|
|
||||||
<div class="bannerIndex mobile02">
|
<div class="bannerIndex mobile02">
|
||||||
<div class="imgBoxIndex">
|
<div class="imgBoxIndex">
|
||||||
|
|||||||
+47
-2
@@ -3476,6 +3476,36 @@ class CommonDatautils {
|
|||||||
return obj;
|
return obj;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
static def queryFirstOrLastRentPrice(city, region, name, sort){
|
||||||
|
DBObject queryCondition = new BasicDBObject()
|
||||||
|
queryCondition.put("city", city);
|
||||||
|
if(region){
|
||||||
|
queryCondition.put("region", region);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(name){
|
||||||
|
queryCondition.put("cat", "district");
|
||||||
|
queryCondition.put("district_name", name)
|
||||||
|
}else{
|
||||||
|
if(region){
|
||||||
|
queryCondition.put("cat", "region")
|
||||||
|
}else{
|
||||||
|
queryCondition.put("cat", "city")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DBCursor cur = SQMETRE_RENT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1)
|
||||||
|
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||||
|
def obj = 0;
|
||||||
|
while(cur.hasNext()) {
|
||||||
|
obj = cur.next()
|
||||||
|
}
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
static def queryCityRegionPrice(city, region, startDate, endDate){
|
static def queryCityRegionPrice(city, region, startDate, endDate){
|
||||||
DBObject queryCondition = new BasicDBObject()
|
DBObject queryCondition = new BasicDBObject()
|
||||||
queryCondition.put("city", city);
|
queryCondition.put("city", city);
|
||||||
@@ -3691,11 +3721,26 @@ class CommonDatautils {
|
|||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static def findSqmetreRent(def city, def region, def name, def weekend){
|
static def findSqmetreRent(def city, def region, def name, def weekend){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
q.put("region", region)
|
if(region){
|
||||||
q.put("district_name", name)
|
q.put("region", region)
|
||||||
|
}
|
||||||
|
if(name){
|
||||||
|
q.put("district_name", name)
|
||||||
|
q.put("cat", "district")
|
||||||
|
}else{
|
||||||
|
if(!region){
|
||||||
|
if(city){
|
||||||
|
q.put("cat", "city")
|
||||||
|
}else{
|
||||||
|
q.put("cat", "region")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
q.put("weekend", weekend)
|
q.put("weekend", weekend)
|
||||||
def dbcur = SQMETRE_RENT_COLL.find(q)
|
def dbcur = SQMETRE_RENT_COLL.find(q)
|
||||||
def dbo = null
|
def dbo = null
|
||||||
|
|||||||
+66
-2
@@ -2,20 +2,30 @@ package com.fangjia.script.gj.data_handle
|
|||||||
|
|
||||||
import com.fangjia.core.util.DateUtil
|
import com.fangjia.core.util.DateUtil
|
||||||
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
|
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
|
||||||
|
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.util.CommonUtils
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.MongoClient
|
||||||
|
|
||||||
class AddCityRegionPrice {
|
class AddCityRegionPrice {
|
||||||
|
|
||||||
|
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new AddCityRegionPrice()
|
def task = new AddCityRegionPrice()
|
||||||
// task.addRegionPrice()
|
// task.addRegionPrice()
|
||||||
// task.addCityprice()
|
// task.addCityprice()
|
||||||
task.addDistrictprice()
|
// task.addDistrictprice()
|
||||||
|
// task.addCityRentPrice()
|
||||||
|
task.addDistrictRentprice()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def addRegionPrice(){
|
def addRegionPrice(){
|
||||||
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
|
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
|
||||||
def citys = CommonDatautils.findAllCity()
|
def citys = ["文山州"]//CommonDatautils.findAllCity()
|
||||||
def add = 0
|
def add = 0
|
||||||
for(def c : citys){
|
for(def c : citys){
|
||||||
def regions = CommonDatautils.getCityRegions(c)
|
def regions = CommonDatautils.getCityRegions(c)
|
||||||
@@ -94,4 +104,58 @@ class AddCityRegionPrice {
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def addCityRentPrice(){
|
||||||
|
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
for(def c : citys){
|
||||||
|
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, 1)
|
||||||
|
def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, -1)
|
||||||
|
if(startObj && endObj){
|
||||||
|
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
|
||||||
|
months.each {m->
|
||||||
|
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
|
||||||
|
def obj = CommonDatautils.findSqmetreRent(c, null, null, lastDay)
|
||||||
|
if(!obj){
|
||||||
|
inserBatch.saveCityCalcCommand(c, null, null, lastDay, "fangjia-priceanalysis-0")
|
||||||
|
println "[add] "+c+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def addDistrictRentprice(){
|
||||||
|
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
for(def c : citys){
|
||||||
|
def buffer = []
|
||||||
|
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||||
|
for(def d : districts){
|
||||||
|
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, 1)
|
||||||
|
// def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, -1)
|
||||||
|
if(startObj){
|
||||||
|
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
|
||||||
|
months.each {m->
|
||||||
|
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
|
||||||
|
def obj = CommonDatautils.findSqmetreRent(c, d.region, d.name, lastDay)
|
||||||
|
if(!obj){
|
||||||
|
def cmd = inserBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, lastDay, "fangjia-priceanalysis-0")
|
||||||
|
buffer << cmd
|
||||||
|
if(buffer.size() == 1000){
|
||||||
|
COMMAND_COLL.insert(buffer)
|
||||||
|
buffer.clear()
|
||||||
|
}
|
||||||
|
println "[add] "+c+" "+ d.name+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!buffer.isEmpty()){
|
||||||
|
COMMAND_COLL.insert(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-4
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
|
|||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
import com.fangjia.script.util.CommonUtils
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DataUtil
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
@@ -34,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
|
|
||||||
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
|
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
|
||||||
|
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new ThirdPriceMappingBatch(), args);
|
runBatch(new ThirdPriceMappingBatch(), args);
|
||||||
@@ -76,7 +77,17 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
address = address + nmd.district
|
address = address + nmd.district
|
||||||
}
|
}
|
||||||
address = address + nmd.name
|
address = address + nmd.name
|
||||||
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true)
|
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false)
|
||||||
|
if(!jsonObject){
|
||||||
|
if(DataUtil.isPreciseAddress(nmd.address)){
|
||||||
|
address = nmd.city
|
||||||
|
if(nmd.district){
|
||||||
|
address = address + nmd.district
|
||||||
|
}
|
||||||
|
address = address + nmd.address
|
||||||
|
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false)
|
||||||
|
}
|
||||||
|
}
|
||||||
if(jsonObject) {
|
if(jsonObject) {
|
||||||
def udata = new BasicDBObject()
|
def udata = new BasicDBObject()
|
||||||
udata.put("match_id",jsonObject.get("id"))
|
udata.put("match_id",jsonObject.get("id"))
|
||||||
@@ -161,7 +172,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
def addOldMatchResult(){
|
def addOldMatchResult(){
|
||||||
def citys = ["上海"]//CommonUtils.findAllCity()
|
def citys = CommonUtils.findAllCity()
|
||||||
def keys = new BasicDBObject()
|
def keys = new BasicDBObject()
|
||||||
keys.put("_id", 1)
|
keys.put("_id", 1)
|
||||||
|
|
||||||
@@ -199,7 +210,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
|
|
||||||
println addCount.get()+"/"+total +" [旧匹配找到] "
|
println addCount.get()+"/"+total +" [旧匹配找到] "
|
||||||
}else{
|
}else{
|
||||||
println addCount.get()+"/"+total +" [旧匹配没找到] "
|
println addCount.get()+"/"+total +" [旧匹配没找到] "+d._id.toString()
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace()
|
e.printStackTrace()
|
||||||
|
|||||||
+13
-6
@@ -22,6 +22,7 @@ import com.mongodb.DBCollection
|
|||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
import com.mongodb.QueryOperators
|
import com.mongodb.QueryOperators
|
||||||
import net.sf.json.JSONObject
|
import net.sf.json.JSONObject
|
||||||
|
import org.apache.commons.lang3.SerializationUtils
|
||||||
import org.apache.commons.lang3.StringUtils
|
import org.apache.commons.lang3.StringUtils
|
||||||
import org.apache.tools.ant.util.DateUtils
|
import org.apache.tools.ant.util.DateUtils
|
||||||
import org.bson.types.ObjectId
|
import org.bson.types.ObjectId
|
||||||
@@ -61,7 +62,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
private static def districtMap = [:];
|
private static def districtMap = [:];
|
||||||
private static def limit = 5000;
|
private static def limit = 5000;
|
||||||
|
|
||||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||||
|
|
||||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
|
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
|
||||||
@@ -92,7 +93,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
cityMap.each{city,alias->
|
cityMap.each{city,alias->
|
||||||
println "开始加载【${city}】"
|
println "开始加载【${city}】"
|
||||||
// if(city == "福州"){
|
// if(city == "上海"){
|
||||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||||
districtMap.putAll(detailMap)
|
districtMap.putAll(detailMap)
|
||||||
// }
|
// }
|
||||||
@@ -463,7 +464,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
// }catch (Exception e){
|
// }catch (Exception e){
|
||||||
// addr = data.address;
|
// addr = data.address;
|
||||||
// }
|
// }
|
||||||
def copyObj = data
|
def copyObj = SerializationUtils.clone(data)
|
||||||
// copyObj.address = addr
|
// copyObj.address = addr
|
||||||
district = getMatchAddressFromMap(copyObj)
|
district = getMatchAddressFromMap(copyObj)
|
||||||
if(district){
|
if(district){
|
||||||
@@ -492,7 +493,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
if(!district){
|
if(!district){
|
||||||
def copyObj = data
|
def copyObj = SerializationUtils.clone(data)
|
||||||
copyObj.address = addr
|
copyObj.address = addr
|
||||||
district = getOpenApiMatch(copyObj)
|
district = getOpenApiMatch(copyObj)
|
||||||
if(!district && addr){
|
if(!district && addr){
|
||||||
@@ -906,7 +907,13 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
def getOpenApiMatch(def data){
|
def getOpenApiMatch(def data){
|
||||||
if(StringUtils.isNotEmpty(data.name)){
|
if(StringUtils.isNotEmpty(data.name)){
|
||||||
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
|
def jsonObject = null
|
||||||
|
if(DataUtil.isPreciseAddress(data.address)){
|
||||||
|
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false)
|
||||||
|
}else{
|
||||||
|
jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
|
||||||
|
}
|
||||||
|
|
||||||
if(jsonObject && jsonObject.credit>0 ){
|
if(jsonObject && jsonObject.credit>0 ){
|
||||||
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
|
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
|
||||||
if(dbo){
|
if(dbo){
|
||||||
@@ -923,7 +930,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def findAllNoMatchData(){
|
def findAllNoMatchData(){
|
||||||
DBObject qdata = new BasicDBObject();
|
DBObject qdata = new BasicDBObject();
|
||||||
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
|
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
|
||||||
// qdata.put("city","福州")
|
// qdata.put("city","上海")
|
||||||
// qdata.put("source","安居客")
|
// qdata.put("source","安居客")
|
||||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||||
|
|||||||
Reference in New Issue
Block a user