diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 660b685bf..f710a70fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1892,6 +1892,33 @@ class CommonDatautils { } + public static def queryFirstOrLastDistrictPrice(city, region, name, sort){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def dbo = null; + while(cur.hasNext()) { + dbo = cur.next() + } + return dbo; + } + + + public static def queryDistrictPriceByWeekend(city, region, name, weekend){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + queryCondition.put("weekend", weekend); + + return DIST_STAT_COLL.findOne(queryCondition) + } + + public static def queryDistrictPrice(city, region, name, weekend, cat){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); @@ -2351,6 +2378,22 @@ class CommonDatautils { } return price; } + + + public static def findCityOrRegionPrice(city, region, weekend){ + DBObject q = new BasicDBObject() + q.put("city", city); + if(region){ + q.put("region", region); + } + q.put('weekend', weekend); + def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1); + def price = null; + for(def obj : dbCursor){ + price = obj.get("avg_price"); + } + return price; + } public static def findDistrictPrice(def city, def region, def name){ DBObject queryCondition = new BasicDBObject() @@ -3414,6 +3457,24 @@ class CommonDatautils { return price; } + + static def queryCityOrRegionPrice(city, region, sort){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + if(region){ + queryCondition.put("region", region); + queryCondition.put("cat", "区域") + }else{ + queryCondition.put("cat", "城市") + } + DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } static def queryCityRegionPrice(city, region, startDate, endDate){ DBObject queryCondition = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy new file mode 100644 index 000000000..667f06d6d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -0,0 +1,97 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.averager.InsertMonitorCalcCommandBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils + +class AddCityRegionPrice { + static main(args) { + def task = new AddCityRegionPrice() +// task.addRegionPrice() +// task.addCityprice() + task.addDistrictprice() + } + + def addRegionPrice(){ + InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() + def citys = CommonDatautils.findAllCity() + def add = 0 + for(def c : citys){ + def regions = CommonDatautils.getCityRegions(c) + for(def r : regions){ + def startObj = CommonDatautils.queryCityOrRegionPrice(c, r, 1) + def endObj = CommonDatautils.queryCityOrRegionPrice(c, r, -1) + if(startObj && endObj){ + def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); + for(def weekend : betweenDates ){ + def price = CommonDatautils.findCityOrRegionPrice(c, r, weekend) + if(!price){ + add++ + insertBatch.saveCityCalcCommand(c,r,null, weekend, "fangjia-priceanalysis-0") + println "[add] ${add} "+c+" "+r+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + } + } + }else{ + println c+" "+r+" 有空值" + } + + } + } + + } + + + def addCityprice(){ + InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() + def citys = CommonDatautils.findAllCity().reverse() + def add = 0 + for(def c : citys){ + def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1) + def endObj = CommonDatautils.queryCityOrRegionPrice(c, null, -1) + if(startObj && endObj){ + def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); + for(def weekend : betweenDates ){ + def price = CommonDatautils.findCityOrRegionPrice(c, null, weekend) + if(!price){ + add++ + insertBatch.saveCityCalcCommand(c,null,null, weekend, "fangjia-priceanalysis-0") + println "[add] ${add} "+c+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + } + } + println c +" 校验完成" + }else{ + println c+" 有空值" + } + } + } + + + def addDistrictprice(){ + InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() + def citys = CommonDatautils.findAllCity().reverse() + def add = 0 + for(def c : citys){ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) + for(def d : districts){ + def startObj = CommonDatautils.queryFirstOrLastDistrictPrice(c, d.region, d.name, 1) + def endObj = CommonDatautils.queryFirstOrLastDistrictPrice(c, d.region, d.name, -1) + if(startObj && endObj){ + def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend); + for(def weekend : betweenDates ){ + def price = CommonDatautils.queryDistrictPriceByWeekend(c, d.region, d.name, weekend) + if(!price){ + add++ + insertBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, weekend, "fangjia-priceanalysis-0") + println "[add] ${add} "+c+" "+d.region+" "+d.name+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd") + } + } + println c +" 校验完成" + }else{ + println c+ " "+d.region+" "+d.name+" 无价格" + } + } + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 636e2b0dc..56915936b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -39,11 +39,14 @@ class DevProcess { DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + + final DBCollection tempJiangsuQiliangColl = MongoUtils.getCollectionByName("crawl", "temp_jiangsu_qiliang", "192.168.0.206",27020) + static main(args) { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -60,7 +63,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() - task.synchDevTo100() +// task.synchDevTo100() // task.addNewDev2() // task.addAlias() @@ -83,6 +86,7 @@ class DevProcess { // task.matchDevShape()//匹配开发区围栏 +// task.addQiDevResult() //获取齐梁数据结果 } @@ -438,20 +442,20 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> - if(!d['I'] && d['J']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d-> + if(!d['B'] && d['A']){ def q = new BasicDBObject() -// q.put("name", d['J']) +// q.put("name", d['A']) // q.put("city",d['G']) -// q.put('alias', new BasicDBObject('$in',[d['J']])) - def name = d['J'].replace('经济技术开发区','经济开发区') +// q.put('alias', new BasicDBObject('$in',[d['A']])) + def name = d['A'].replace('经济技术开发区','经济开发区') q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 def obj = development_zone_CONN.findOne(q) if(obj){ d['I'] = "库中有开发区-相似2" - d['K'] = obj.name +// d['K'] = obj.name // d['C'] = obj._id.toString() -// d['D'] = obj.name + d['B'] = obj.name // if(obj.name == d['A']){ // d['E'] = "相同" // }else{ @@ -461,7 +465,7 @@ class DevProcess { // d['F'] = obj.name } } - },0); + },1); } def calAddrDistince(){ @@ -1120,4 +1124,28 @@ class DevProcess { } }); } + + def addQiDevResult(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d-> + if(d['E'] && !d['Z']){ + def q = new BasicDBObject("qiye_address", d['E']) + def obj = tempJiangsuQiliangColl.findOne(q) + if(obj){ + def qlmap = obj.ql + if(qlmap){ + d['Z'] = "有" + if(qlmap.indusParkList){ + def parklist = qlmap.indusParkList + def names = [] + for(def p : parklist){ + names << p.indusParkName + } + d['W'] = names.join(",") + d['X'] = parklist.size() + } + } + } + } + },0) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 5fefd22f6..1c94475bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -36,8 +36,8 @@ class MinglujiData { // exportDatas(city) // matchDev(prov, city) // matchCommunity(city) -// fiveLevelSplit(city) - nineLevelSplit(city) + fiveLevelSplit(city) +// nineLevelSplit(city) // tempMatchDev(prov) // checkDev() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index 6b186692a..4009417a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils @@ -19,13 +20,19 @@ import org.apache.commons.lang.math.RandomUtils import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class ImportIndexStats extends BaseBatch{ + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; //public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index'); public static final DBCollection on_line = MongoUtils.getCollectionByName("fangjia", "stats_index", "mongo.db.fangjia.com") public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") - public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") + public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl) final DBCollection indices_price_house_sellColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell", "192.168.0.206", 27020) final DBCollection indices_price_house_sell_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell_type", "192.168.0.206", 27020) @@ -33,6 +40,7 @@ class ImportIndexStats extends BaseBatch{ final DBCollection indices_price_house_newColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new", "192.168.0.206", 27020) final DBCollection indices_price_house_new_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new_type", "192.168.0.206", 27020) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) //DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater") private static def cityList=["北京", "天津", "石家庄", "太原", "呼和浩特", "沈阳", "大连", "长春", "哈尔滨", "上海", "南京", "杭州", "宁波", "合肥", "福州", "厦门", @@ -74,80 +82,114 @@ class ImportIndexStats extends BaseBatch{ def s = "201601" def startDate = null; def stop = false; + def runDates = [] while(!stop){ startDate = DateUtils.parseFromTightYYYYMM(s); def year = startDate.getYear()+1900; def month = startDate.month+1; def date = createDateString(year, month, 0, ""); + + def map = [:] + map.year = year + map.month = month + map.date = date + runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") - if(year==2024 && month==9){ + if(year==2024 && month==8){ stop = true; } - calculatePriceIndex(year, month); - println "运行......"+date+" 完成" } - - + for(def map : runDates){ + calculatePriceIndex(map.year, map.month, map.date); + } } + //计算统计局没有发布的城市的价格指数 - def calculatePriceIndex(def year, def month){ - def citys =CommonDatautils.findAllCity(); - def date = createDateString(year, month, 0, ""); + def calculatePriceIndex(def year, def month, def date){ + def citys = CommonDatautils.findAllCity(); +// def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); - def index = 0; - if(citys){ - for(def city : citys){ - if(haveIndexCitys.contains(city)){ - continue; - } - - def id = findPriceIndex(city, date.toInteger()); - if(id){//已计算 - continue; - } - - def result = [:]; - calculateAvgPriceIndex(city, year, month, result); - calculateAreaPriceInfex(result, city, year, month, 90);//0-90 - calculateAreaPriceInfex(result, city, year, month, 144);//90-144 - calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000 + def totalCount = new AtomicLong() + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(citys.size()) - -// checkPriceIndex(result);//涨跌幅控制 - index++; - if(result){ - result.city = city; - result.yearmonth = date.toInteger(); - result.source = 2; - result.remark = "均价计算"; - result.cat = "district"; - result.type = "城市" - savePriceIndex(result); - println "${index}"+ result+" "; - }else{ - println "${index}"+ result+" "+city +" 无数据"; - } + def newCitys = [] + for(def i=0;i def temp = v - 100; if(k == "markup" || k == "areaMarkup90" || k == "areaMarkup144" || k == "areaMarkup1000"){ - if(temp < -2 || temp > 1){ - v = getNewMarkup(temp, -2, 1) + 100; + if(temp < -3 || temp > 3){ + v = getNewMarkup(temp, -3, 3) + 100; detailFlag = 2; } }else if(k == "yearMarkup" || k == "areaYearMarkup90" || k == "areaYearMarkup144" || k == "areaYearMarkup1000"){ - if(temp < -5 || temp > 7){ - v = getNewMarkup(temp, -5, 7) + 100; + if(temp < -20 || temp > 30){ + v = getNewMarkup(temp, -20, 30) + 100; detailFlag = 2; } }else{ - if(temp < -6 || temp > 29){ - v = getNewMarkup(temp, -6, 29) + 100; + if(temp < -50 || temp > 100){ + v = getNewMarkup(temp, -50, 100) + 100; detailFlag = 2; } } @@ -483,8 +529,7 @@ class ImportIndexStats extends BaseBatch{ def findCityByDates(def city, def startDate, def endDate){ DBObject q = new BasicDBObject() q.put("city", city); - q.put("region", null); - q.put("block", null); + q.put("cat", "城市") if(startDate && endDate){ q.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate))); }