update
This commit is contained in:
+9
-5
@@ -144,16 +144,17 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
def street = ""
|
def street = ""
|
||||||
def streetNumber = ""
|
def streetNumber = ""
|
||||||
if(trade.address){
|
if(trade.address){
|
||||||
if(trade.address.toString().endsWith("弄")){
|
if(trade.address.toString().contains("弄")){
|
||||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||||
if(number){
|
if(number){
|
||||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||||
streetNumber = number
|
streetNumber = number
|
||||||
}
|
}
|
||||||
}else{
|
}
|
||||||
|
if(!street){
|
||||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||||
if(jsonObj && jsonObj.searchAddress){
|
if(jsonObj && jsonObj.searchAddress){
|
||||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("弄")){
|
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||||
if(number){
|
if(number){
|
||||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||||
@@ -170,7 +171,7 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
|
|
||||||
if(!street){
|
if(!street){
|
||||||
if(obj.address){
|
if(obj.address){
|
||||||
if(obj.address.toString().endsWith("弄")){
|
if(obj.address.toString().contains("弄")){
|
||||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||||
if(number){
|
if(number){
|
||||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||||
@@ -282,7 +283,7 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
if(trade.address){
|
if(trade.address){
|
||||||
if(trade.address.toString().endsWith("弄")){
|
if(trade.address.toString().contains("弄")){
|
||||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||||
if(number){
|
if(number){
|
||||||
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||||
@@ -420,6 +421,9 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
if(dist.city=="上海"){
|
if(dist.city=="上海"){
|
||||||
startPrice = (int)(price - price*0.1)
|
startPrice = (int)(price - price*0.1)
|
||||||
endPrice = (int)(price*0.5 + price)
|
endPrice = (int)(price*0.5 + price)
|
||||||
|
}else if(dist.city=="杭州"){
|
||||||
|
startPrice = (int)(price - price*0.2)
|
||||||
|
endPrice = (int)(price*0.5 + price)
|
||||||
}else{
|
}else{
|
||||||
startPrice = (int)(price - price*0.5)
|
startPrice = (int)(price - price*0.5)
|
||||||
endPrice = (int)(price*0.5 + price)
|
endPrice = (int)(price*0.5 + price)
|
||||||
|
|||||||
+1
-1
@@ -66,7 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def citys = ["东莞","中山","文昌","保亭","陵水"]//CcbCommonUtils.citys;
|
def citys = CcbCommonUtils.citys;
|
||||||
def index = 0
|
def index = 0
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
|
|||||||
+1
-1
@@ -187,7 +187,7 @@ class CcbCommonUtils {
|
|||||||
q.put("visible",0)
|
q.put("visible",0)
|
||||||
q.put("cat", cat);
|
q.put("cat", cat);
|
||||||
q.put("city", city);
|
q.put("city", city);
|
||||||
// q.put("name", "石泉六村")
|
// q.put("name", "莱兴公寓")
|
||||||
// q.put("name_cat", 0);
|
// q.put("name_cat", 0);
|
||||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||||
|
|
||||||
|
|||||||
+103
-2
@@ -2,6 +2,11 @@ package com.fangjia.script.ccbevaluation
|
|||||||
|
|
||||||
import com.fangjia.core.util.HttpRequest
|
import com.fangjia.core.util.HttpRequest
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
import net.sf.json.JSONObject
|
import net.sf.json.JSONObject
|
||||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||||
|
|
||||||
@@ -26,16 +31,112 @@ class UpdateHouseRegion {
|
|||||||
}
|
}
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||||
|
|
||||||
|
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new UpdateHouseRegion()
|
def task = new UpdateHouseRegion()
|
||||||
task.run()
|
// task.updateSeawaterRegion()
|
||||||
|
// task.updateDealRegion()
|
||||||
|
task.updateThirdPartyRegion()
|
||||||
}
|
}
|
||||||
def run(){
|
|
||||||
|
def updateThirdPartyRegion(){
|
||||||
|
def city = "杭州"
|
||||||
|
def rets = findThirdPartyPrice(city)
|
||||||
|
batchThirdPartyUpdate(city, rets);
|
||||||
|
|
||||||
|
}
|
||||||
|
def findThirdPartyPrice(city){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city",city)
|
||||||
|
q.put("s_date",new BasicDBObject('$gte', 202101))
|
||||||
|
def dbcursor = thirdPartyColl.find(q)
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcursor){
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateDealRegion(){
|
||||||
|
def city = "杭州"
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01")));
|
||||||
|
def cursor = DEAL_PRICE_COLL.find(q)
|
||||||
|
def index = 0
|
||||||
|
def updateIndex = 0
|
||||||
|
for(def obj : cursor){
|
||||||
|
index++
|
||||||
|
def region = obj.region
|
||||||
|
def name = obj.district_name
|
||||||
|
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||||
|
if (!rets) {
|
||||||
|
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rets.size() == 1) {
|
||||||
|
def dist = rets[0]
|
||||||
|
if (region != dist.region) {
|
||||||
|
updateIndex++
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("region", dist.region)
|
||||||
|
DEAL_PRICE_COLL.update(new BasicDBObject('_id', obj.get("_id")), new BasicDBObject('$set', udata))
|
||||||
|
println updateIndex+" "+index+" "+region+" "+dist.region
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
def updateSeawaterRegion(){
|
||||||
def city = "杭州"
|
def city = "杭州"
|
||||||
def rets = findHouseSource(city, 20190701, 20191231)
|
def rets = findHouseSource(city, 20190701, 20191231)
|
||||||
batchUpdate(city, rets);
|
batchUpdate(city, rets);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def batchThirdPartyUpdate(def city, def thirdDatas){
|
||||||
|
def updateCount = new AtomicLong()
|
||||||
|
def curCount = new AtomicLong()
|
||||||
|
def totalCount = thirdDatas.size();
|
||||||
|
final CountDownLatch latch = new CountDownLatch(totalCount)
|
||||||
|
println city +" 第三方总量:"+totalCount
|
||||||
|
for(def i=0;i<totalCount;i++) {
|
||||||
|
final def data = thirdDatas[i];
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
curCount.addAndGet(1)
|
||||||
|
try {
|
||||||
|
def region = data.region
|
||||||
|
def name = data.name
|
||||||
|
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||||
|
if (!rets) {
|
||||||
|
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rets.size() == 1) {
|
||||||
|
def dist = rets[0]
|
||||||
|
if (region != dist.region) {
|
||||||
|
updateCount.addAndGet(1)
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("region", dist.region)
|
||||||
|
thirdPartyColl.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
|
||||||
|
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
def findCount(def city, def startDate, def endDate){
|
def findCount(def city, def startDate, def endDate){
|
||||||
final StringBuffer sqlBuffer = new StringBuffer()
|
final StringBuffer sqlBuffer = new StringBuffer()
|
||||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1")
|
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1")
|
||||||
|
|||||||
Reference in New Issue
Block a user