From 28440961f6b3107d13b9822a553c1e5155e9d89c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 11:10:11 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E5=A4=9A=E7=BA=BF=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/trade/ImportIndexStats.groovy | 160 +++++++++++------- 1 file changed, 101 insertions(+), 59 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index 6b186692a..a1d5b4f9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils @@ -19,6 +20,10 @@ import org.apache.commons.lang.math.RandomUtils import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class ImportIndexStats extends BaseBatch{ @@ -33,6 +38,7 @@ class ImportIndexStats extends BaseBatch{ final DBCollection indices_price_house_newColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new", "192.168.0.206", 27020) final DBCollection indices_price_house_new_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new_type", "192.168.0.206", 27020) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) //DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater") private static def cityList=["北京", "天津", "石家庄", "太原", "呼和浩特", "沈阳", "大连", "长春", "哈尔滨", "上海", "南京", "杭州", "宁波", "合肥", "福州", "厦门", @@ -74,80 +80,112 @@ class ImportIndexStats extends BaseBatch{ def s = "201601" def startDate = null; def stop = false; + def runDates = [] while(!stop){ startDate = DateUtils.parseFromTightYYYYMM(s); def year = startDate.getYear()+1900; def month = startDate.month+1; def date = createDateString(year, month, 0, ""); + + def map = [:] + map.year = year + map.month = month + runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") - if(year==2024 && month==9){ + if(year==2024 && month==8){ stop = true; } - calculatePriceIndex(year, month); - println "运行......"+date+" 完成" } - - + for(def map : runDates){ + calculatePriceIndex(map.year, map.month); + } } + //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month){ def citys =CommonDatautils.findAllCity(); def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); - def index = 0; - if(citys){ - for(def city : citys){ - if(haveIndexCitys.contains(city)){ - continue; - } - - def id = findPriceIndex(city, date.toInteger()); - if(id){//已计算 - continue; - } - - def result = [:]; - calculateAvgPriceIndex(city, year, month, result); - calculateAreaPriceInfex(result, city, year, month, 90);//0-90 - calculateAreaPriceInfex(result, city, year, month, 144);//90-144 - calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000 + def totalCount = new AtomicLong() + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(citys.size()) - -// checkPriceIndex(result);//涨跌幅控制 - index++; - if(result){ - result.city = city; - result.yearmonth = date.toInteger(); - result.source = 2; - result.remark = "均价计算"; - result.cat = "district"; - result.type = "城市" - savePriceIndex(result); - println "${index}"+ result+" "; - }else{ - println "${index}"+ result+" "+city +" 无数据"; - } + def newCitys = [] + for(def i=0;i def temp = v - 100; if(k == "markup" || k == "areaMarkup90" || k == "areaMarkup144" || k == "areaMarkup1000"){ - if(temp < -2 || temp > 1){ - v = getNewMarkup(temp, -2, 1) + 100; + if(temp < -3 || temp > 3){ + v = getNewMarkup(temp, -3, 3) + 100; detailFlag = 2; } }else if(k == "yearMarkup" || k == "areaYearMarkup90" || k == "areaYearMarkup144" || k == "areaYearMarkup1000"){ - if(temp < -5 || temp > 7){ - v = getNewMarkup(temp, -5, 7) + 100; + if(temp < -20 || temp > 30){ + v = getNewMarkup(temp, -20, 30) + 100; detailFlag = 2; } }else{ - if(temp < -6 || temp > 29){ - v = getNewMarkup(temp, -6, 29) + 100; + if(temp < -50 || temp > 100){ + v = getNewMarkup(temp, -50, 100) + 100; detailFlag = 2; } } From f5b483ef93425fb4ad7665e5bdd7f721c906c4ae Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 11:35:04 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E6=9C=88=E4=BB=BD=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/trade/ImportIndexStats.groovy | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index a1d5b4f9c..6ef0094ba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -23,6 +23,7 @@ import java.sql.ResultSet import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) @@ -90,6 +91,7 @@ class ImportIndexStats extends BaseBatch{ def map = [:] map.year = year map.month = month + map.date = date runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") @@ -98,14 +100,14 @@ class ImportIndexStats extends BaseBatch{ } } for(def map : runDates){ - calculatePriceIndex(map.year, map.month); + calculatePriceIndex(map.year, map.month, map.date); } } //计算统计局没有发布的城市的价格指数 - def calculatePriceIndex(def year, def month){ + def calculatePriceIndex(def year, def month, def date){ def citys =CommonDatautils.findAllCity(); - def date = createDateString(year, month, 0, ""); +// def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); def totalCount = new AtomicLong() @@ -130,7 +132,7 @@ class ImportIndexStats extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - def result = calculating(city, year, month); + def result = calculating(city, year, month, date); println result println "${city}[${totalCount.get()}/${total}]运行......"+year+"-"+month+" 完成" @@ -142,11 +144,10 @@ class ImportIndexStats extends BaseBatch{ } }); } - latch.await() - return addCount.get(); + latch.await(5, TimeUnit.MINUTES); } - def calculating(city, year, month){ + def calculating(city, year, month, date){ def result = [:]; calculateAvgPriceIndex(city, year, month, result); @@ -157,7 +158,7 @@ class ImportIndexStats extends BaseBatch{ checkPriceIndex(result);//涨跌幅控制 if(result){ result.city = city; - result.yearmonth = (year.toString()+month.toString()).toInteger(); + result.yearmonth = date; result.source = 2; result.remark = "均价计算"; result.cat = "district"; From 2e9de3eaa6812edea182c9d3986c101e722bbcb1 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 11:42:08 +0800 Subject: [PATCH 03/10] update --- .../groovy/com/fangjia/script/trade/ImportIndexStats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index 6ef0094ba..9e4af1c91 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -158,7 +158,7 @@ class ImportIndexStats extends BaseBatch{ checkPriceIndex(result);//涨跌幅控制 if(result){ result.city = city; - result.yearmonth = date; + result.yearmonth = date.toInteger(); result.source = 2; result.remark = "均价计算"; result.cat = "district"; From 1fb09940051f6d5193707aeb6137bebefd0d2c91 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 13:45:33 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E8=A1=A5=E5=85=85=E5=8C=BA=E5=9F=9F?= =?UTF-8?q?=E4=BB=B7=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 34 ++++++++++++++++++ .../gj/data_handle/AddCityRegionPrice.groovy | 36 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 660b685bf..f6dc6e3b1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -2351,6 +2351,22 @@ class CommonDatautils { } return price; } + + + public static def findCityOrRegionPrice(city, region, weekend){ + DBObject q = new BasicDBObject() + q.put("city", city); + if(region){ + q.put("region", region); + } + q.put('weekend', weekend); + def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1); + def price = null; + for(def obj : dbCursor){ + price = obj.get("avg_price"); + } + return price; + } public static def findDistrictPrice(def city, def region, def name){ DBObject queryCondition = new BasicDBObject() @@ -3414,6 +3430,24 @@ class CommonDatautils { return price; } + + static def queryCityOrRegionPrice(city, region, sort){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + if(region){ + queryCondition.put("region", region); + queryCondition.put("cat", "区域") + }else{ + queryCondition.put("cat", "城市") + } + DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } static def queryCityRegionPrice(city, region, startDate, endDate){ DBObject queryCondition = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy new file mode 100644 index 000000000..6fc2eae33 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.averager.InsertMonitorCalcCommandBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils + +class AddCityRegionPrice { + static main(args) { + def task = new AddCityRegionPrice() + task.run() + } + + def run(){ + InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() + def citys = CommonDatautils.findAllCity() + def add = 0 + for(def c : citys){ + def regions = CommonDatautils.getCityRegions(c) + for(def r : regions){ + def startObj = CommonDatautils.queryCityOrRegionPrice(c, r, 1) + def endObj = CommonDatautils.queryCityOrRegionPrice(c, r, -1) + def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); + for(def weekend : betweenDates ){ + def price = CommonDatautils.findCityOrRegionPrice(c, r, weekend) + if(!price){ + add++ + insertBatch.saveCityCalcCommand(c,r,null, weekend, "fangjia-priceanalysis-0") + println "[add] ${add} "+c+" "+r+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + } + } + } + } + + } +} From be273c5f7ce0d50cc92742f613e2ad70c19be8ec Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 14:08:01 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E7=A9=BA=E5=80=BC=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddCityRegionPrice.groovy | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index 6fc2eae33..816bf86c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -20,15 +20,20 @@ class AddCityRegionPrice { for(def r : regions){ def startObj = CommonDatautils.queryCityOrRegionPrice(c, r, 1) def endObj = CommonDatautils.queryCityOrRegionPrice(c, r, -1) - def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); - for(def weekend : betweenDates ){ - def price = CommonDatautils.findCityOrRegionPrice(c, r, weekend) - if(!price){ - add++ - insertBatch.saveCityCalcCommand(c,r,null, weekend, "fangjia-priceanalysis-0") - println "[add] ${add} "+c+" "+r+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + if(startObj && endObj){ + def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); + for(def weekend : betweenDates ){ + def price = CommonDatautils.findCityOrRegionPrice(c, r, weekend) + if(!price){ + add++ + insertBatch.saveCityCalcCommand(c,r,null, weekend, "fangjia-priceanalysis-0") + println "[add] ${add} "+c+" "+r+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + } } + }else{ + println c+" "+r+" 有空值" } + } } From d58d6313d459d0377751a24166420578216d7553 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 14:44:19 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E9=80=89=E8=BE=BD=E5=AE=81=E7=9C=81?= =?UTF-8?q?=E8=B7=91=E4=B9=9D=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/DevProcess.groovy | 48 +++++++++++++++---- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 636e2b0dc..56915936b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -39,11 +39,14 @@ class DevProcess { DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + + final DBCollection tempJiangsuQiliangColl = MongoUtils.getCollectionByName("crawl", "temp_jiangsu_qiliang", "192.168.0.206",27020) + static main(args) { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -60,7 +63,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() - task.synchDevTo100() +// task.synchDevTo100() // task.addNewDev2() // task.addAlias() @@ -83,6 +86,7 @@ class DevProcess { // task.matchDevShape()//匹配开发区围栏 +// task.addQiDevResult() //获取齐梁数据结果 } @@ -438,20 +442,20 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> - if(!d['I'] && d['J']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d-> + if(!d['B'] && d['A']){ def q = new BasicDBObject() -// q.put("name", d['J']) +// q.put("name", d['A']) // q.put("city",d['G']) -// q.put('alias', new BasicDBObject('$in',[d['J']])) - def name = d['J'].replace('经济技术开发区','经济开发区') +// q.put('alias', new BasicDBObject('$in',[d['A']])) + def name = d['A'].replace('经济技术开发区','经济开发区') q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 def obj = development_zone_CONN.findOne(q) if(obj){ d['I'] = "库中有开发区-相似2" - d['K'] = obj.name +// d['K'] = obj.name // d['C'] = obj._id.toString() -// d['D'] = obj.name + d['B'] = obj.name // if(obj.name == d['A']){ // d['E'] = "相同" // }else{ @@ -461,7 +465,7 @@ class DevProcess { // d['F'] = obj.name } } - },0); + },1); } def calAddrDistince(){ @@ -1120,4 +1124,28 @@ class DevProcess { } }); } + + def addQiDevResult(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d-> + if(d['E'] && !d['Z']){ + def q = new BasicDBObject("qiye_address", d['E']) + def obj = tempJiangsuQiliangColl.findOne(q) + if(obj){ + def qlmap = obj.ql + if(qlmap){ + d['Z'] = "有" + if(qlmap.indusParkList){ + def parklist = qlmap.indusParkList + def names = [] + for(def p : parklist){ + names << p.indusParkName + } + d['W'] = names.join(",") + d['X'] = parklist.size() + } + } + } + } + },0) + } } From e619d868b045c3a9a6df70db3b4920290fd83599 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 16:50:32 +0800 Subject: [PATCH 07/10] =?UTF-8?q?=E6=97=A0=E5=AE=9A=E5=9F=BA=EF=BC=8C?= =?UTF-8?q?=E6=97=A0=E7=8E=AF=E6=AF=94=EF=BC=8C=E6=97=A0=E5=90=8C=E6=AF=94?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E7=AE=97=E4=B8=8D=E5=90=8C=E9=9D=A2=E7=A7=AF?= =?UTF-8?q?=E6=8C=87=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/trade/ImportIndexStats.groovy | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index 9e4af1c91..aa6426963 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -28,10 +28,11 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class ImportIndexStats extends BaseBatch{ + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; //public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index'); public static final DBCollection on_line = MongoUtils.getCollectionByName("fangjia", "stats_index", "mongo.db.fangjia.com") public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") - public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") + public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl) final DBCollection indices_price_house_sellColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell", "192.168.0.206", 27020) final DBCollection indices_price_house_sell_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell_type", "192.168.0.206", 27020) @@ -39,7 +40,7 @@ class ImportIndexStats extends BaseBatch{ final DBCollection indices_price_house_newColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new", "192.168.0.206", 27020) final DBCollection indices_price_house_new_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new_type", "192.168.0.206", 27020) - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) //DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater") private static def cityList=["北京", "天津", "石家庄", "太原", "呼和浩特", "沈阳", "大连", "长春", "哈尔滨", "上海", "南京", "杭州", "宁波", "合肥", "福州", "厦门", @@ -106,7 +107,7 @@ class ImportIndexStats extends BaseBatch{ //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month, def date){ - def citys =CommonDatautils.findAllCity(); + def citys = ["三沙"]//CommonDatautils.findAllCity(); // def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); @@ -144,26 +145,28 @@ class ImportIndexStats extends BaseBatch{ } }); } - latch.await(5, TimeUnit.MINUTES); + latch.await(2, TimeUnit.MINUTES); } def calculating(city, year, month, date){ def result = [:]; calculateAvgPriceIndex(city, year, month, result); - calculateAreaPriceInfex(result, city, year, month, 90);//0-90 - calculateAreaPriceInfex(result, city, year, month, 144);//90-144 - calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000 - - checkPriceIndex(result);//涨跌幅控制 if(result){ - result.city = city; - result.yearmonth = date.toInteger(); - result.source = 2; - result.remark = "均价计算"; - result.cat = "district"; - result.type = "城市" - savePriceIndex(result); + calculateAreaPriceInfex(result, city, year, month, 90);//0-90 + calculateAreaPriceInfex(result, city, year, month, 144);//90-144 + calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000 + + checkPriceIndex(result);//涨跌幅控制 + if(result){ + result.city = city; + result.yearmonth = date.toInteger(); + result.source = 2; + result.remark = "均价计算"; + result.cat = "district"; + result.type = "城市" + savePriceIndex(result); + } } return result } @@ -526,8 +529,7 @@ class ImportIndexStats extends BaseBatch{ def findCityByDates(def city, def startDate, def endDate){ DBObject q = new BasicDBObject() q.put("city", city); - q.put("region", null); - q.put("block", null); + q.put("cat", "城市") if(startDate && endDate){ q.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate))); } From 8dba6af26d4ed6166ce7ec0430d65f86367f102f Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 16:55:50 +0800 Subject: [PATCH 08/10] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/trade/ImportIndexStats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index aa6426963..426860962 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -107,7 +107,7 @@ class ImportIndexStats extends BaseBatch{ //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month, def date){ - def citys = ["三沙"]//CommonDatautils.findAllCity(); + def citys = CommonDatautils.findAllCity(); // def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); From 3c50796f8d0588ac43326d0db38523893ca4f093 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 17:04:44 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E5=88=86=E6=8D=A2=E6=88=90=E7=A7=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/trade/ImportIndexStats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index 426860962..4009417a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -145,7 +145,7 @@ class ImportIndexStats extends BaseBatch{ } }); } - latch.await(2, TimeUnit.MINUTES); + latch.await(20, TimeUnit.SECONDS); } def calculating(city, year, month, date){ From b679ffa1a43015a37259d91edbaa388a31177b99 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Sun, 29 Sep 2024 17:48:15 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E5=90=8D=E5=BD=95=E9=9B=86=E8=B7=915?= =?UTF-8?q?=E7=BA=A7=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 27 +++++++++ .../gj/data_handle/AddCityRegionPrice.groovy | 60 ++++++++++++++++++- .../script/gj/data_handle/MinglujiData.groovy | 4 +- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index f6dc6e3b1..f710a70fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1892,6 +1892,33 @@ class CommonDatautils { } + public static def queryFirstOrLastDistrictPrice(city, region, name, sort){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def dbo = null; + while(cur.hasNext()) { + dbo = cur.next() + } + return dbo; + } + + + public static def queryDistrictPriceByWeekend(city, region, name, weekend){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + queryCondition.put("weekend", weekend); + + return DIST_STAT_COLL.findOne(queryCondition) + } + + public static def queryDistrictPrice(city, region, name, weekend, cat){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index 816bf86c9..667f06d6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -8,10 +8,12 @@ import com.fangjia.script.util.CommonUtils class AddCityRegionPrice { static main(args) { def task = new AddCityRegionPrice() - task.run() +// task.addRegionPrice() +// task.addCityprice() + task.addDistrictprice() } - def run(){ + def addRegionPrice(){ InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() def citys = CommonDatautils.findAllCity() def add = 0 @@ -38,4 +40,58 @@ class AddCityRegionPrice { } } + + + def addCityprice(){ + InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() + def citys = CommonDatautils.findAllCity().reverse() + def add = 0 + for(def c : citys){ + def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1) + def endObj = CommonDatautils.queryCityOrRegionPrice(c, null, -1) + if(startObj && endObj){ + def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); + for(def weekend : betweenDates ){ + def price = CommonDatautils.findCityOrRegionPrice(c, null, weekend) + if(!price){ + add++ + insertBatch.saveCityCalcCommand(c,null,null, weekend, "fangjia-priceanalysis-0") + println "[add] ${add} "+c+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + } + } + println c +" 校验完成" + }else{ + println c+" 有空值" + } + } + } + + + def addDistrictprice(){ + InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() + def citys = CommonDatautils.findAllCity().reverse() + def add = 0 + for(def c : citys){ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) + for(def d : districts){ + def startObj = CommonDatautils.queryFirstOrLastDistrictPrice(c, d.region, d.name, 1) + def endObj = CommonDatautils.queryFirstOrLastDistrictPrice(c, d.region, d.name, -1) + if(startObj && endObj){ + def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); + for(def weekend : betweenDates ){ + def price = CommonDatautils.queryDistrictPriceByWeekend(c, d.region, d.name, weekend) + if(!price){ + add++ + insertBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, weekend, "fangjia-priceanalysis-0") + println "[add] ${add} "+c+" "+d.region+" "+d.name+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + } + } + println c +" 校验完成" + }else{ + println c+ " "+d.region+" "+d.name+" 无价格" + } + } + } + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 5fefd22f6..1c94475bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -36,8 +36,8 @@ class MinglujiData { // exportDatas(city) // matchDev(prov, city) // matchCommunity(city) -// fiveLevelSplit(city) - nineLevelSplit(city) + fiveLevelSplit(city) +// nineLevelSplit(city) // tempMatchDev(prov) // checkDev()