Merge branch 'script_gj' into 'master'

update

See merge request !123
This commit is contained in:
2021-09-22 09:39:59 +08:00
3 changed files with 74 additions and 41 deletions
@@ -37,13 +37,13 @@ class UpdateHouseRegion {
static main(args) {
def task = new UpdateHouseRegion()
// task.updateSeawaterRegion()
task.updateSeawaterRegion()
// task.updateDealRegion()
task.updateThirdPartyRegion()
// task.updateThirdPartyRegion()
}
def updateThirdPartyRegion(){
def city = "杭州"
def city = "东莞"
def rets = findThirdPartyPrice(city)
batchThirdPartyUpdate(city, rets);
@@ -60,40 +60,70 @@ class UpdateHouseRegion {
return rets;
}
def updateDealRegion(){
def city = "杭州"
def findDealData(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01")));
def cursor = DEAL_PRICE_COLL.find(q)
def cursor = DEAL_PRICE_COLL.find(q).batchSize(2000)
def datas = []
def index = 0
def updateIndex = 0
for(def obj : cursor){
index++
def region = obj.region
def name = obj.district_name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateIndex++
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', obj.get("_id")), new BasicDBObject('$set', udata))
println updateIndex+" "+index+" "+region+" "+dist.region
}
}
datas << obj
println index
}
return datas
}
def updateDealRegion(){
def city = "东莞"
def datas = findDealData(city)
batchDealRegion(city, datas)
}
def batchDealRegion(def city, def datas){
def updateCount = new AtomicLong()
def curCount = new AtomicLong()
def totalCount = datas.size();
final CountDownLatch latch = new CountDownLatch(totalCount)
println city +" 成交总量:"+totalCount
for(def i=0;i<totalCount;i++) {
final def data = datas[i];
fixedThreadPool.execute(new Runnable() {
void run() {
curCount.addAndGet(1)
try {
def region = data.region
def name = data.district_name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
}
def updateSeawaterRegion(){
def city = "杭州"
def rets = findHouseSource(city, 20190701, 20191231)
def city = "东莞"
def rets = findHouseSource(city, 20190701, 20211001)
batchUpdate(city, rets);
}
@@ -135,6 +165,7 @@ class UpdateHouseRegion {
}
})
}
latch.await()
}
def findCount(def city, def startDate, def endDate){
@@ -187,6 +218,7 @@ class UpdateHouseRegion {
})
}
latch.await()
}
def findHouseSource(def city, def startDate, def endDate){
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-07-01"
static def statDate = "2021-08-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年9月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -299,14 +299,14 @@ class YouDaDataParser {
for(def i=0;i<datas.size();i++){
index++;
def data = datas[i];
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
// count_.incrementAndGet()
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.incrementAndGet()
saveEsfHouseToDB(data);
// }
// }
// );
}
}
);
}
latch.await()
}
@@ -37,10 +37,11 @@ class SeaweedDataMatchProcess {
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
}
if(!rMap && address){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
if(rMap && rMap.region && !region.contains(rMap.region)){
rMap = null
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
}
}