update
This commit is contained in:
+60
-28
@@ -37,13 +37,13 @@ class UpdateHouseRegion {
|
||||
|
||||
static main(args) {
|
||||
def task = new UpdateHouseRegion()
|
||||
// task.updateSeawaterRegion()
|
||||
task.updateSeawaterRegion()
|
||||
// task.updateDealRegion()
|
||||
task.updateThirdPartyRegion()
|
||||
// task.updateThirdPartyRegion()
|
||||
}
|
||||
|
||||
def updateThirdPartyRegion(){
|
||||
def city = "杭州"
|
||||
def city = "东莞"
|
||||
def rets = findThirdPartyPrice(city)
|
||||
batchThirdPartyUpdate(city, rets);
|
||||
|
||||
@@ -60,40 +60,70 @@ class UpdateHouseRegion {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def updateDealRegion(){
|
||||
def city = "杭州"
|
||||
def findDealData(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01")));
|
||||
def cursor = DEAL_PRICE_COLL.find(q)
|
||||
def cursor = DEAL_PRICE_COLL.find(q).batchSize(2000)
|
||||
def datas = []
|
||||
def index = 0
|
||||
def updateIndex = 0
|
||||
for(def obj : cursor){
|
||||
index++
|
||||
def region = obj.region
|
||||
def name = obj.district_name
|
||||
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||
if (!rets) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||
}
|
||||
|
||||
if (rets.size() == 1) {
|
||||
def dist = rets[0]
|
||||
if (region != dist.region) {
|
||||
updateIndex++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", dist.region)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject('_id', obj.get("_id")), new BasicDBObject('$set', udata))
|
||||
println updateIndex+" "+index+" "+region+" "+dist.region
|
||||
}
|
||||
}
|
||||
datas << obj
|
||||
println index
|
||||
}
|
||||
|
||||
|
||||
return datas
|
||||
}
|
||||
|
||||
def updateDealRegion(){
|
||||
def city = "东莞"
|
||||
def datas = findDealData(city)
|
||||
batchDealRegion(city, datas)
|
||||
}
|
||||
|
||||
def batchDealRegion(def city, def datas){
|
||||
def updateCount = new AtomicLong()
|
||||
def curCount = new AtomicLong()
|
||||
def totalCount = datas.size();
|
||||
final CountDownLatch latch = new CountDownLatch(totalCount)
|
||||
println city +" 成交总量:"+totalCount
|
||||
for(def i=0;i<totalCount;i++) {
|
||||
final def data = datas[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
curCount.addAndGet(1)
|
||||
try {
|
||||
def region = data.region
|
||||
def name = data.district_name
|
||||
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||
if (!rets) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||
}
|
||||
|
||||
if (rets.size() == 1) {
|
||||
def dist = rets[0]
|
||||
if (region != dist.region) {
|
||||
updateCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", dist.region)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
|
||||
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def updateSeawaterRegion(){
|
||||
def city = "杭州"
|
||||
def rets = findHouseSource(city, 20190701, 20191231)
|
||||
def city = "东莞"
|
||||
def rets = findHouseSource(city, 20190701, 20211001)
|
||||
batchUpdate(city, rets);
|
||||
}
|
||||
|
||||
@@ -135,6 +165,7 @@ class UpdateHouseRegion {
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def findCount(def city, def startDate, def endDate){
|
||||
@@ -187,6 +218,7 @@ class UpdateHouseRegion {
|
||||
})
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def findHouseSource(def city, def startDate, def endDate){
|
||||
|
||||
+10
-10
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-07-01"
|
||||
static def statDate = "2021-08-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年9月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
@@ -299,14 +299,14 @@ class YouDaDataParser {
|
||||
for(def i=0;i<datas.size();i++){
|
||||
index++;
|
||||
def data = datas[i];
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
// count_.incrementAndGet()
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.incrementAndGet()
|
||||
saveEsfHouseToDB(data);
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
+4
-3
@@ -37,10 +37,11 @@ class SeaweedDataMatchProcess {
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
||||
}
|
||||
|
||||
|
||||
if(!rMap && address){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
if(rMap && rMap.region && !region.contains(rMap.region)){
|
||||
rMap = null
|
||||
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user